@embedder/embedder 1.0.7

package/bundle/repomap-python/repomap_server.py

@@ -0,0 +1,585 @@
+import asyncio
+import json
+import os
+import sys
+import logging
+from pathlib import Path
+from typing import List, Optional, Dict, Any, Set
+import dataclasses
+
+# Check for debug mode from command line arguments or environment variable
+DEBUG_MODE = '--debug' in sys.argv or os.getenv('REPOMAP_DEBUG', '').lower() in ('true', '1', 'yes')
+
+# Add startup logging before any imports that might fail (only if debug mode)
+if DEBUG_MODE:
+    startup_log = logging.getLogger('startup')
+    startup_handler = logging.StreamHandler(sys.stderr)
+    startup_handler.setLevel(logging.INFO)
+    startup_formatter = logging.Formatter('🐍 STARTUP: %(message)s')
+    startup_handler.setFormatter(startup_formatter)
+    startup_log.addHandler(startup_handler)
+    startup_log.setLevel(logging.INFO)
+
+    startup_log.info("Starting RepoMapper server initialization...")
+    startup_log.info(f"Python version: {sys.version}")
+    startup_log.info(f"Working directory: {os.getcwd()}")
+    startup_log.info(f"Script path: {__file__}")
+    startup_log.info(f"Python path: {sys.path[:3]}...")  # Show first few entries
+
+try:
+    if DEBUG_MODE:
+        startup_log.info("Importing FastMCP...")
+    from fastmcp import FastMCP, settings
+    if DEBUG_MODE:
+        startup_log.info("✅ FastMCP imported successfully")
+except ImportError as e:
+    if DEBUG_MODE:
+        startup_log.error(f"❌ Failed to import FastMCP: {e}")
+        startup_log.error("💡 Try: pip install fastmcp")
+    sys.exit(1)
+
+try:
+    if DEBUG_MODE:
+        startup_log.info("Importing RepoMapper dependencies...")
+    from repomap_class import RepoMap
+    from utils import count_tokens, read_text
+    from scm import get_scm_fname
+    from importance import filter_important_files
+    if DEBUG_MODE:
+        startup_log.info("✅ All RepoMapper dependencies imported successfully")
+except ImportError as e:
+    if DEBUG_MODE:
+        startup_log.error(f"❌ Failed to import RepoMapper dependencies: {e}")
+        startup_log.error(f"💡 Missing dependency: {str(e).split('No module named')[-1] if 'No module named' in str(e) else 'unknown'}")
+        startup_log.error(f"Current working directory: {os.getcwd()}")
+        startup_log.error(f"Files in current directory: {list(os.listdir('.'))[:10]}")
+    sys.exit(1)
+
+
+# Helper function from your CLI, useful to have here
+def find_src_files(directory: str) -> List[str]:
+    if not os.path.isdir(directory):
+        return [directory] if os.path.isfile(directory) else []
+    src_files = []
+    for r, d, f_list in os.walk(directory):
+        d[:] = [d_name for d_name in d if not d_name.startswith('.') and d_name not in {'node_modules', '__pycache__', 'venv', 'env'}]
+        for f in f_list:
+            if not f.startswith('.'):
+                src_files.append(os.path.join(r, f))
+    return src_files
+
+# Configure logging - show info and above
+root_logger = logging.getLogger()
+root_logger.setLevel(logging.INFO)
+
+# Create console handler for info and above (explicitly use stderr)
+console_handler = logging.StreamHandler(sys.stderr)
+console_handler.setLevel(logging.INFO)
+console_formatter = logging.Formatter('%(levelname)-5s %(asctime)-15s %(name)s:%(funcName)s:%(lineno)d - %(message)s')
+console_handler.setFormatter(console_formatter)
+root_logger.addHandler(console_handler)
+
+# Suppress FastMCP logs
+fastmcp_logger = logging.getLogger('fastmcp')
+fastmcp_logger.setLevel(logging.ERROR)
+# Suppress server startup message
+server_logger = logging.getLogger('fastmcp.server')
+server_logger.setLevel(logging.ERROR)
+
+log = logging.getLogger(__name__)
+
+# Set global stateless_http setting
+settings.stateless_http = True
+
+# Create MCP server
+mcp = FastMCP("RepoMapServer")
+
+async def _repo_map_impl(
+    project_root: str,
+    chat_files: Optional[List[str]] = None,
+    other_files: Optional[List[str]] = None,
+    token_limit: Any = 8192,  # Accept any type to handle empty strings
+    exclude_unranked: bool = False,
+    force_refresh: bool = False,
+    mentioned_files: Optional[List[str]] = None,
+    mentioned_idents: Optional[List[str]] = None,
+    verbose: bool = False,
+    max_context_window: Optional[int] = None,
+) -> Dict[str, Any]:
+    """Generate a repository map for the specified files, providing a list of function prototypes and variables for files as well as relevant related
+    files. Provide filenames relative to the project_root. In addition to the files provided, relevant related files will also be included with a
+    very small ranking boost.
+
+    :param project_root: Root directory of the project to search.  (must be an absolute path!)
+    :param chat_files: A list of file paths that are currently in the chat context. These files will receive the highest ranking.
+    :param other_files: A list of other relevant file paths in the repository to consider for the map. They receive a lower ranking boost than mentioned_files and chat_files.
+    :param token_limit: The maximum number of tokens the generated repository map should occupy. Defaults to 8192.
+    :param exclude_unranked: If True, files with a PageRank of 0.0 will be excluded from the map. Defaults to False.
+    :param force_refresh: If True, forces a refresh of the repository map cache. Defaults to False.
+    :param mentioned_files: Optional list of file paths explicitly mentioned in the conversation and receive a mid-level ranking boost.
+    :param mentioned_idents: Optional list of identifiers explicitly mentioned in the conversation, to boost their ranking.
+    :param verbose: If True, enables verbose logging for the RepoMap generation process. Defaults to False.
+    :param max_context_window: Optional maximum context window size for token calculation, used to adjust map token limit when no chat files are provided.
+    :returns: A dictionary containing:
+        - 'map': the generated repository map string
+        - 'report': a dictionary with file processing details including:
+            - 'included': list of processed files
+            - 'excluded': dictionary of excluded files with reasons
+            - 'definition_matches': count of matched definitions
+            - 'reference_matches': count of matched references
+            - 'total_files_considered': total files processed
+        Or an 'error' key if an error occurred.
+    """
+    if not os.path.isdir(project_root):
+        return {"error": f"Project root directory not found: {project_root}"}
+
+    # 1. Handle and validate parameters
+    # Convert token_limit to integer with fallback
+    try:
+        token_limit = int(token_limit) if token_limit else 8192
+    except (TypeError, ValueError):
+        token_limit = 8192
+    
+    # Ensure token_limit is positive
+    if token_limit <= 0:
+        token_limit = 8192
+    
+    chat_files_list = chat_files or []
+    mentioned_fnames_set = set(mentioned_files) if mentioned_files else None
+    mentioned_idents_set = set(mentioned_idents) if mentioned_idents else None
+
+    # 2. If a specific list of other_files isn't provided, scan the whole root directory.
+    # This should happen regardless of whether chat_files are present.
+    effective_other_files = []
+    if other_files:
+        effective_other_files = other_files
+    else:
+        log.info("No other_files provided, scanning root directory for context...")
+        effective_other_files = find_src_files(project_root)
+
+    # Add a print statement for debugging so you can see what the tool is working with.
+    log.debug(f"Chat files: {chat_files_list}")
+    log.debug(f"Effective other_files count: {len(effective_other_files)}")
+
+    # If after all that we have no files, we can exit early.
+    if not chat_files_list and not effective_other_files:
+        log.info("No files to process.")
+        return {"map": "No files found to generate a map."}
+
+    # 3. Resolve paths relative to project root
+    root_path = Path(project_root).resolve()
+    abs_chat_files = [str(root_path / f) for f in chat_files_list]
+    abs_other_files = [str(root_path / f) for f in effective_other_files]
+    
+    # Remove any chat files from the other_files list to avoid duplication
+    abs_chat_files_set = set(abs_chat_files)
+    abs_other_files = [f for f in abs_other_files if f not in abs_chat_files_set]
+
+    # 4. Instantiate and run RepoMap
+    try:
+        repo_mapper = RepoMap(
+            map_tokens=token_limit,
+            root=str(root_path),
+            token_counter_func=lambda text: count_tokens(text, "gpt-4"),
+            file_reader_func=read_text,
+            output_handler_funcs={'info': log.info, 'warning': log.warning, 'error': log.error},
+            verbose=verbose,
+            exclude_unranked=exclude_unranked,
+            max_context_window=max_context_window
+        )
+    except Exception as e:
+        log.exception(f"Failed to initialize RepoMap for project '{project_root}': {e}")
+        return {"error": f"Failed to initialize RepoMap: {str(e)}"}
+
+    try:
+        map_content, file_report = await asyncio.to_thread(
+            repo_mapper.get_repo_map,
+            chat_files=abs_chat_files,
+            other_files=abs_other_files,
+            mentioned_fnames=mentioned_fnames_set,
+            mentioned_idents=mentioned_idents_set,
+            force_refresh=force_refresh
+        )
+        
+        # Convert FileReport to dictionary for JSON serialization
+        report_dict = {
+            "excluded": file_report.excluded,
+            "definition_matches": file_report.definition_matches,
+            "reference_matches": file_report.reference_matches,
+            "total_files_considered": file_report.total_files_considered
+        }
+        
+        return {
+            "map": map_content or "No repository map could be generated.",
+            "report": report_dict
+        }
+    except Exception as e:
+        log.exception(f"Error generating repository map for project '{project_root}': {e}")
+        return {"error": f"Error generating repository map: {str(e)}"}
+
+@mcp.tool()
+async def repo_map(
+    project_root: str,
+    chat_files: Optional[List[str]] = None,
+    other_files: Optional[List[str]] = None,
+    token_limit: Any = 8192,  # Accept any type to handle empty strings
+    exclude_unranked: bool = False,
+    force_refresh: bool = False,
+    mentioned_files: Optional[List[str]] = None,
+    mentioned_idents: Optional[List[str]] = None,
+    verbose: bool = False,
+    max_context_window: Optional[int] = None,
+) -> Dict[str, Any]:
+    """Generate a repository map for a given project root.
+    
+    Returns:
+        Dictionary containing map content and generation report
+    """
+    return await _repo_map_impl(project_root, chat_files, other_files, token_limit, exclude_unranked, force_refresh, mentioned_files, mentioned_idents, verbose, max_context_window)
+    
+async def _search_identifiers_impl(
+    project_root: str,
+    query: str,
+    max_results: int = 50,
+    context_lines: int = 2,
+    include_definitions: bool = True,
+    include_references: bool = True
+) -> Dict[str, Any]:
+    """Implementation of search_identifiers without MCP decoration for direct calling.
+    
+    Args:
+        project_root: Root directory of the project to search.  (must be an absolute path!)
+        query: Search query (identifier name)
+        max_results: Maximum number of results to return
+        context_lines: Number of lines of context to show
+        include_definitions: Whether to include definition occurrences
+        include_references: Whether to include reference occurrences
+    
+    Returns:
+        Dictionary containing search results or error message
+    """
+    if not os.path.isdir(project_root):
+        return {"error": f"Project root directory not found: {project_root}"}
+
+    try:
+        # Initialize RepoMap with search-specific settings
+        repo_map = RepoMap(
+            root=project_root,
+            token_counter_func=lambda text: count_tokens(text, "gpt-4"),
+            file_reader_func=read_text,
+            output_handler_funcs={'info': log.info, 'warning': log.warning, 'error': log.error},
+            verbose=False,
+            exclude_unranked=True
+        )
+
+        # Find all source files in the project
+        all_files = find_src_files(project_root)
+        
+        # Get all tags (definitions and references) for all files
+        all_tags = []
+        for file_path in all_files:
+            rel_path = str(Path(file_path).relative_to(project_root))
+            tags = repo_map.get_tags(file_path, rel_path)
+            all_tags.extend(tags)
+
+        # Filter tags based on search query and options
+        matching_tags = []
+        query_lower = query.lower()
+        
+        for tag in all_tags:
+            if query_lower in tag.name.lower():
+                if (tag.kind == "def" and include_definitions) or \
+                   (tag.kind == "ref" and include_references):
+                    matching_tags.append(tag)
+
+        # Sort by relevance (definitions first, then references)
+        matching_tags.sort(key=lambda x: (x.kind != "def", x.name.lower().find(query_lower)))
+
+        # Limit results
+        matching_tags = matching_tags[:max_results]
+
+        # Format results with context
+        results = []
+        for tag in matching_tags:
+            file_path = str(Path(project_root) / tag.rel_fname)
+            
+            # Calculate context range based on context_lines parameter
+            start_line = max(1, tag.line - context_lines)
+            end_line = tag.line + context_lines
+            context_range = list(range(start_line, end_line + 1))
+            
+            context = repo_map.render_tree(
+                file_path,
+                tag.rel_fname,
+                context_range
+            )
+            
+            if context:
+                results.append({
+                    "file": tag.rel_fname,
+                    "line": tag.line,
+                    "name": tag.name,
+                    "kind": tag.kind,
+                    "context": context
+                })
+
+        return {"results": results}
+
+    except Exception as e:
+        log.exception(f"Error searching identifiers in project '{project_root}': {e}")
+        return {"error": f"Error searching identifiers: {str(e)}"}    
+
+@mcp.tool()
+async def search_identifiers(
+    project_root: str,
+    query: str,
+    max_results: int = 50,
+    context_lines: int = 2,
+    include_definitions: bool = True,
+    include_references: bool = True
+) -> Dict[str, Any]:
+    """Search for identifiers in code files. Get back a list of matching identifiers with their file, line number, and context.
+       When searching, just use the identifier name without any special characters, prefixes or suffixes. The search is 
+       case-insensitive.
+
+    Args:
+        project_root: Root directory of the project to search.  (must be an absolute path!)
+        query: Search query (identifier name)
+        max_results: Maximum number of results to return
+        context_lines: Number of lines of context to show
+        include_definitions: Whether to include definition occurrences
+        include_references: Whether to include reference occurrences
+    
+    Returns:
+        Dictionary containing search results or error message
+    """
+    return await _search_identifiers_impl(project_root, query, max_results, context_lines, include_definitions, include_references)
+
+async def _warm_up_cache_impl(project_root: str) -> Dict[str, Any]:
+    """Implementation of warm_up_cache without MCP decoration for direct calling.
+    
+    Performs background indexing of the project to populate the cache.
+    This should be called when the application starts to avoid delays
+    when the user first runs search queries.
+    
+    Args:
+        project_root: Root directory of the project to index
+    
+    Returns:
+        Dictionary containing status or error message
+    """
+    if not os.path.isdir(project_root):
+        return {"error": f"Project root directory not found: {project_root}"}
+
+    try:
+        log.info(f"Starting background cache warm-up for project: {project_root}")
+        
+        # Initialize RepoMap with indexing-specific settings
+        repo_map = RepoMap(
+            root=project_root,
+            token_counter_func=lambda text: count_tokens(text, "gpt-4"),
+            file_reader_func=read_text,
+            output_handler_funcs={'info': log.info, 'warning': log.warning, 'error': log.error},
+            verbose=False,
+            exclude_unranked=True
+        )
+
+        # Find all source files in the project
+        all_files = find_src_files(project_root)
+        log.info(f"Found {len(all_files)} files to index")
+        
+        # Process files in batches to avoid overwhelming the system
+        batch_size = 50
+        indexed_files = 0
+        
+        for i in range(0, len(all_files), batch_size):
+            batch_files = all_files[i:i + batch_size]
+            
+            for file_path in batch_files:
+                try:
+                    rel_path = str(Path(file_path).relative_to(project_root))
+                    # This call to get_tags will populate the cache for this file
+                    tags = repo_map.get_tags(file_path, rel_path)
+                    indexed_files += 1
+                    
+                    # Log progress every 100 files
+                    if indexed_files % 100 == 0:
+                        log.info(f"Cache warm-up progress: {indexed_files}/{len(all_files)} files indexed")
+                        
+                except Exception as file_error:
+                    # Log individual file errors but continue processing
+                    log.warning(f"Failed to index file {file_path}: {file_error}")
+            
+            # Small delay between batches to be nice to the system
+            await asyncio.sleep(0.01)
+
+        log.info(f"Cache warm-up completed. Indexed {indexed_files} files.")
+        
+        return {
+            "status": "completed",
+            "files_indexed": indexed_files,
+            "total_files": len(all_files)
+        }
+
+    except Exception as e:
+        log.exception(f"Error during cache warm-up for project '{project_root}': {e}")
+        return {"error": f"Error during cache warm-up: {str(e)}"}
+
+@mcp.tool()
+async def warm_up_cache(project_root: str) -> Dict[str, Any]:
+    """Warm up the cache by indexing all files in the project.
+    
+    This is a background operation that should be called when the application
+    starts to ensure that subsequent search queries are fast.
+    
+    Args:
+        project_root: Root directory of the project to index (must be an absolute path!)
+    
+    Returns:
+        Dictionary containing status information or error message
+    """
+    return await _warm_up_cache_impl(project_root)
+
+def handle_interactive_query(query_data: Dict[str, Any]) -> Dict[str, Any]:
+    """Handle interactive queries from the TypeScript client"""
+    try:
+        if 'tool' not in query_data:
+            return {"error": "Missing 'tool' field in query"}
+        
+        tool_name = query_data['tool']
+        args = query_data.get('args', {})
+        
+        if tool_name == 'repo_map':
+            return asyncio.run(_repo_map_impl(**args))
+        elif tool_name == 'search_identifiers':
+            return asyncio.run(_search_identifiers_impl(**args))
+        elif tool_name == 'warm_up_cache':
+            return asyncio.run(_warm_up_cache_impl(**args))
+        else:
+            return {"error": f"Unknown tool: {tool_name}"}
+    except Exception as e:
+        log.exception(f"Error handling interactive query: {e}")
+        return {"error": str(e)}
+
+def interactive_mode():
+    """Run in interactive mode for direct TypeScript communication"""
+    log.info("Starting RepoMapper in terminal interactive mode...")
+    
+    try:
+        while True:
+            # Read line from stdin
+            line = input()
+            if not line.strip():
+                continue
+                
+            try:
+                # Parse the JSON query
+                query_data = json.loads(line)
+                query_id = query_data.get('id', 'unknown')
+                query = query_data.get('query', {})
+                
+                # Handle the query
+                result = handle_interactive_query(query)
+                
+                # Send response back
+                response = {
+                    "id": query_id,
+                    "result": result
+                }
+                print(json.dumps(response), flush=True)
+                
+            except json.JSONDecodeError as e:
+                error_response = {
+                    "id": "unknown",
+                    "error": f"Invalid JSON: {str(e)}"
+                }
+                print(json.dumps(error_response), flush=True)
+            except EOFError:
+                # Client disconnected
+                break
+            except Exception as e:
+                error_response = {
+                    "id": query_data.get('id', 'unknown') if 'query_data' in locals() else 'unknown',
+                    "error": str(e)
+                }
+                print(json.dumps(error_response), flush=True)
+                
+    except KeyboardInterrupt:
+        log.debug("Received interrupt signal, shutting down...")
+    except Exception as e:
+        log.exception(f"Fatal error in interactive mode: {e}")
+
+# --- Main Entry Point ---
+def main():
+    import sys
+    import argparse
+    
+    if DEBUG_MODE:
+        startup_log.info("Parsing command line arguments...")
+    parser = argparse.ArgumentParser(description='RepoMapper Server')
+    parser.add_argument('--interactive', action='store_true', 
+                       help='Force interactive mode for direct stdin/stdout JSON communication')
+    parser.add_argument('--mcp-server', action='store_true',
+                       help='Force MCP server mode')
+    parser.add_argument('--debug', action='store_true',
+                       help='Enable debug mode with verbose logging')
+    
+    args = parser.parse_args()
+    if DEBUG_MODE:
+        startup_log.info(f"Arguments: {args}")
+    
+    # Determine which mode to run in
+    if args.interactive:
+        # Explicitly requested interactive mode
+        if DEBUG_MODE:
+            startup_log.info("🔄 Running in interactive mode...")
+        interactive_mode()
+    elif args.mcp_server:
+        # Explicitly requested MCP server mode
+        if DEBUG_MODE:
+            startup_log.info("🚀 Running in MCP server mode...")
+        log.info("Starting FastMCP server...")
+        sys.stderr.flush()  # Force immediate output to stderr
+        
+        try:
+            if DEBUG_MODE:
+                startup_log.info("🔧 Initializing FastMCP server...")
+            mcp.run()
+        except Exception as e:
+            if DEBUG_MODE:
+                startup_log.error(f"❌ FastMCP server failed to start: {e}")
+                startup_log.error(f"Error type: {type(e).__name__}")
+                import traceback
+                startup_log.error(f"Traceback: {traceback.format_exc()}")
+            sys.exit(1)
+    else:
+        # Auto-detect based on stdin
+        if DEBUG_MODE:
+            startup_log.info("🔍 Auto-detecting mode based on stdin...")
+        if sys.stdin.isatty():
+            # Running from terminal - use interactive mode for direct communication
+            if DEBUG_MODE:
+                startup_log.info("📺 TTY detected, using interactive mode")
+            interactive_mode()
+        else:
+            # Running as subprocess - default to interactive mode for TypeScript tools
+            # (This fixes the issue where piped stdin was incorrectly starting MCP server)
+            if DEBUG_MODE:
+                startup_log.info("🔗 Non-TTY detected, using interactive mode for subprocess")
+            interactive_mode()
+
+if __name__ == "__main__":
+    try:
+        if DEBUG_MODE:
+            startup_log.info("🎬 Starting main function...")
+        main()
+    except KeyboardInterrupt:
+        if DEBUG_MODE:
+            startup_log.info("🛑 Interrupted by user")
+        sys.exit(0)
+    except Exception as e:
+        if DEBUG_MODE:
+            startup_log.error(f"💥 Fatal error in main: {e}")
+            import traceback
+            startup_log.error(f"Traceback: {traceback.format_exc()}")
+        sys.exit(1)

package/bundle/repomap-python/requirements.txt

@@ -0,0 +1,7 @@
+tiktoken>=0.5.0
+networkx>=3.0
+diskcache>=5.6.0
+grep-ast>=0.3.0
+tree-sitter>=0.20.0
+pygments>=2.14.0
+fastmcp

package/bundle/repomap-python/scm.py

@@ -0,0 +1,59 @@
+"""
+SCM file handling for RepoMap.
+"""
+
+from pathlib import Path
+from typing import Optional
+
+def get_scm_fname(lang: str) -> Optional[str]:
+    """Get the SCM query file for a language."""
+    scm_files = {
+        'arduino': 'arduino-tags.scm',
+        'chatito': 'chatito-tags.scm',
+        'commonlisp': 'commonlisp-tags.scm',
+        'cpp': 'cpp-tags.scm',
+        'csharp': 'csharp-tags.scm',
+        'c': 'c-tags.scm',
+        'dart': 'dart-tags.scm',
+        'd': 'd-tags.scm',
+        'elisp': 'elisp-tags.scm',
+        'elixir': 'elixir-tags.scm',
+        'elm': 'elm-tags.scm',
+        'gleam': 'gleam-tags.scm',
+        'go': 'go-tags.scm',
+        'javascript': 'javascript-tags.scm',
+        'java': 'java-tags.scm',
+        'lua': 'lua-tags.scm',
+        'ocaml_interface': 'ocaml_interface-tags.scm',
+        'ocaml': 'ocaml-tags.scm',
+        'pony': 'pony-tags.scm',
+        'properties': 'properties-tags.scm',
+        'python': 'python-tags.scm',
+        'racket': 'racket-tags.scm',
+        'r': 'r-tags.scm',
+        'ruby': 'ruby-tags.scm',
+        'rust': 'rust-tags.scm',
+        'solidity': 'solidity-tags.scm',
+        'swift': 'swift-tags.scm',
+        'udev': 'udev-tags.scm',
+        'c_sharp': 'c_sharp-tags.scm',
+        'hcl': 'hcl-tags.scm',
+        'kotlin': 'kotlin-tags.scm',
+        'php': 'php-tags.scm',
+        'ql': 'ql-tags.scm',
+        'scala': 'scala-tags.scm',
+        'typescript': 'typescript-tags.scm',
+    }
+    
+    if lang in scm_files:
+        scm_filename = scm_files[lang]
+        # Search in tree-sitter-language-pack
+        scm_path = Path(__file__).parent / "queries" / "tree-sitter-language-pack" / scm_filename
+        if scm_path.exists():
+            return str(scm_path)
+        # Search in tree-sitter-languages
+        scm_path = Path(__file__).parent / "queries" / "tree-sitter-languages" / scm_filename
+        if scm_path.exists():
+            return str(scm_path)
+    
+    return None