codebase_index 0.2.1 → 0.3.1

data/exe/codebase-index-mcp-http

@@ -16,40 +16,12 @@ require_relative '../lib/codebase_index'
 require_relative '../lib/codebase_index/dependency_graph'
 require_relative '../lib/codebase_index/graph_analyzer'
 require_relative '../lib/codebase_index/mcp/server'
+require_relative '../lib/codebase_index/mcp/bootstrapper'
 require_relative '../lib/codebase_index/embedding/text_preparer'
 require_relative '../lib/codebase_index/embedding/indexer'
 
-index_dir = ARGV[0] || ENV['CODEBASE_INDEX_DIR'] || Dir.pwd
-
-unless Dir.exist?(index_dir)
-  warn "Error: Index directory does not exist: #{index_dir}"
-  exit 1
-end
-
-unless File.exist?(File.join(index_dir, 'manifest.json'))
-  warn "Error: No manifest.json found in: #{index_dir}"
-  warn 'Run `bundle exec rake codebase_index:extract` in your Rails app first.'
-  exit 1
-end
-
-# Attempt to build a retriever for semantic search.
-# Auto-configures from environment variables when no explicit configuration exists.
-retriever = begin
-  config = CodebaseIndex.configuration
-
-  if !config.embedding_provider && ENV.fetch('OPENAI_API_KEY', nil)
-    config.vector_store = :in_memory
-    config.metadata_store = :in_memory
-    config.graph_store = :in_memory
-    config.embedding_provider = :openai
-    config.embedding_options = { api_key: ENV.fetch('OPENAI_API_KEY', nil) }
-  end
-
-  CodebaseIndex::Builder.new(config).build_retriever if config.embedding_provider
-rescue StandardError => e
-  warn "Note: Semantic search unavailable (#{e.message}). Using pattern-based search only."
-  nil
-end
+index_dir = CodebaseIndex::MCP::Bootstrapper.resolve_index_dir(ARGV)
+retriever = CodebaseIndex::MCP::Bootstrapper.build_retriever
 
 port = (ENV['PORT'] || 9292).to_i
 host = ENV['HOST'] || 'localhost'

data/lib/codebase_index/ast/method_extractor.rb

@@ -16,6 +16,8 @@ module CodebaseIndex
     #   # => "def create\n  @user = User.find(params[:id])\nend\n"
     #
     class MethodExtractor
+      include SourceSpan
+
       # @param parser [Ast::Parser, nil] Parser instance (creates default if nil)
       def initialize(parser: nil)
         @parser = parser || Parser.new
@@ -62,14 +64,7 @@ module CodebaseIndex
         return node.source if node.source
 
         # Fallback: extract by line range
-        return nil unless node.line && node.end_line
-
-        lines = source.lines
-        start_idx = node.line - 1
-        end_idx = node.end_line - 1
-        return nil if start_idx.negative? || end_idx >= lines.length
-
-        lines[start_idx..end_idx].join
+        extract_source_span(source, node.line, node.end_line)
       end
     end
   end

data/lib/codebase_index/ast/node.rb

@@ -84,5 +84,33 @@ module CodebaseIndex
         end
       end
     end
+
+    # Mixin for line-range source extraction, shared across Parser, MethodExtractor,
+    # and ClassAnalyzer.
+    #
+    # @example
+    #   include Ast::SourceSpan
+    #   extract_source_span(source, node.line, node.end_line)
+    #
+    module SourceSpan
+      private
+
+      # Extract source lines for a 1-based start/end line range.
+      #
+      # @param source [String] Full source text
+      # @param start_line [Integer, nil] 1-based start line
+      # @param end_line [Integer, nil] 1-based end line
+      # @return [String, nil] Extracted lines joined, or nil if out of range
+      def extract_source_span(source, start_line, end_line)
+        return nil unless start_line && end_line
+
+        lines = source.lines
+        start_idx = start_line - 1
+        end_idx = end_line - 1
+        return nil if start_idx.negative? || end_idx >= lines.length
+
+        lines[start_idx..end_idx].join
+      end
+    end
   end
 end

data/lib/codebase_index/ast/parser.rb

@@ -15,6 +15,8 @@ module CodebaseIndex
     #   root.find_all(:def).first.method_name #=> "bar"
     #
     class Parser
+      include SourceSpan
+
       # Parse Ruby source into a normalized AST.
       #
       # @param source [String] Ruby source code
@@ -26,8 +28,6 @@ module CodebaseIndex
         else
           parse_with_parser_gem(source)
         end
-      rescue CodebaseIndex::ExtractionError
-        raise
       rescue StandardError => e
         raise CodebaseIndex::ExtractionError, "Failed to parse source: #{e.message}"
       end
@@ -36,15 +36,12 @@ module CodebaseIndex
       #
       # @return [Boolean]
       def prism_available?
-        if @prism_available.nil?
-          begin
-            require 'prism'
-            @prism_available = defined?(Prism) ? true : false
-          rescue LoadError
-            @prism_available = false
-          end
-        end
-        @prism_available
+        return @prism_available unless @prism_available.nil?
+
+        require 'prism'
+        @prism_available = true
+      rescue LoadError
+        @prism_available = false
       end
 
       private
@@ -105,10 +102,8 @@ module CodebaseIndex
           )
         when Prism::ConstantPathNode
           convert_prism_constant_path(prism_node, source)
-        when Prism::IfNode
+        when Prism::IfNode, Prism::UnlessNode
           convert_prism_if(prism_node, source)
-        when Prism::UnlessNode
-          convert_prism_unless(prism_node, source)
         when Prism::CaseNode
           convert_prism_case(prism_node, source)
         when Prism::BeginNode
@@ -223,17 +218,7 @@ module CodebaseIndex
       def convert_prism_class(prism_node, source)
         name_node = convert_prism_node(prism_node.constant_path, source)
         superclass = prism_node.superclass ? convert_prism_node(prism_node.superclass, source) : nil
-        body_children = if prism_node.body
-                          if prism_node.body.is_a?(Prism::StatementsNode)
-                            prism_node.body.body.map do |c|
-                              convert_prism_node(c, source)
-                            end
-                          else
-                            [convert_prism_node(prism_node.body, source)]
-                          end
-                        else
-                          []
-                        end
+        body_children = extract_prism_body_children(prism_node, source)
 
         children = [name_node, superclass] + body_children
 
@@ -248,17 +233,7 @@ module CodebaseIndex
 
       def convert_prism_module(prism_node, source)
         name_node = convert_prism_node(prism_node.constant_path, source)
-        body_children = if prism_node.body
-                          if prism_node.body.is_a?(Prism::StatementsNode)
-                            prism_node.body.body.map do |c|
-                              convert_prism_node(c, source)
-                            end
-                          else
-                            [convert_prism_node(prism_node.body, source)]
-                          end
-                        else
-                          []
-                        end
+        body_children = extract_prism_body_children(prism_node, source)
 
         children = [name_node] + body_children
 
@@ -272,15 +247,7 @@ module CodebaseIndex
       end
 
       def convert_prism_def(prism_node, source)
-        body_children = if prism_node.body
-                          if prism_node.body.is_a?(Prism::StatementsNode)
-                            prism_node.body.body.map { |c| convert_prism_node(c, source) }
-                          else
-                            [convert_prism_node(prism_node.body, source)]
-                          end
-                        else
-                          []
-                        end
+        body_children = extract_prism_body_children(prism_node, source)
 
         is_class_method = prism_node.respond_to?(:receiver) && prism_node.receiver
         receiver_text = if is_class_method
@@ -346,13 +313,14 @@ module CodebaseIndex
 
       def convert_prism_constant_path(prism_node, _source)
         parent_text = (extract_const_path_text(prism_node.parent) if prism_node.parent)
+        const_name = prism_node.respond_to?(:name) ? prism_node.name.to_s : prism_node.child.name.to_s
 
         Node.new(
           type: :const,
           children: [],
           line: line_for_prism(prism_node),
           receiver: parent_text,
-          method_name: prism_node.name.to_s
+          method_name: const_name
         )
       end
 
@@ -364,23 +332,8 @@ module CodebaseIndex
         end
 
         then_body = prism_node.statements ? convert_prism_node(prism_node.statements, source) : nil
-        else_body = prism_node.subsequent ? convert_prism_node(prism_node.subsequent, source) : nil
-
-        Node.new(
-          type: :if,
-          children: [condition, then_body, else_body].compact,
-          line: line_for_prism(prism_node),
-          end_line: end_line_for_prism(prism_node),
-          source: condition_source
-        )
-      end
-
-      def convert_prism_unless(prism_node, source)
-        condition = convert_prism_node(prism_node.predicate, source)
-        condition_source = extract_prism_source_text(prism_node.predicate, source)
-
-        then_body = prism_node.statements ? convert_prism_node(prism_node.statements, source) : nil
-        else_body = prism_node.else_clause ? convert_prism_node(prism_node.else_clause, source) : nil
+        else_clause = prism_else_clause(prism_node)
+        else_body = else_clause ? convert_prism_node(else_clause, source) : nil
 
         Node.new(
           type: :if,
@@ -395,10 +348,21 @@ module CodebaseIndex
         children = []
         children << convert_prism_node(prism_node.predicate, source) if prism_node.predicate
         prism_node.conditions.each { |c| children << convert_prism_node(c, source) }
-        children << convert_prism_node(prism_node.else_clause, source) if prism_node.else_clause
+        else_clause = prism_else_clause(prism_node)
+        children << convert_prism_node(else_clause, source) if else_clause
         Node.new(type: :case, children: children, line: line_for_prism(prism_node))
       end
 
+      def extract_prism_body_children(prism_node, source)
+        return [] unless prism_node.body
+
+        if prism_node.body.is_a?(Prism::StatementsNode)
+          prism_node.body.body.map { |c| convert_prism_node(c, source) }
+        else
+          [convert_prism_node(prism_node.body, source)]
+        end
+      end
+
       def convert_prism_children(statements_node, source)
         return [] unless statements_node
 
@@ -410,12 +374,19 @@ module CodebaseIndex
       end
 
       def extract_prism_generic_children(prism_node, source)
-        children = []
-        prism_node.child_nodes.compact.each do |child|
-          converted = convert_prism_node(child, source)
-          children << converted if converted
+        prism_node.child_nodes.compact.filter_map { |child| convert_prism_node(child, source) }
+      end
+
+      # Portable accessor for the else/consequent clause of if/unless/case nodes.
+      # Prism < 1.0 uses :consequent, Prism >= 1.0 uses :else_clause/:subsequent.
+      def prism_else_clause(node)
+        if node.respond_to?(:consequent)
+          node.consequent
+        elsif node.respond_to?(:else_clause)
+          node.else_clause
+        elsif node.respond_to?(:subsequent)
+          node.subsequent
         end
-        children
       end
 
       def line_for_prism(node)
@@ -427,12 +398,7 @@ module CodebaseIndex
       end
 
       def extract_prism_source_span(node, source)
-        lines = source.lines
-        start_idx = node.location.start_line - 1
-        end_idx = node.location.end_line - 1
-        return nil if start_idx.negative? || end_idx >= lines.length
-
-        lines[start_idx..end_idx].join
+        extract_source_span(source, node.location.start_line, node.location.end_line)
       end
 
       def extract_prism_source_text(node, source)
@@ -463,17 +429,13 @@ module CodebaseIndex
           node.name.to_s
         when Prism::ConstantPathNode
           parent = node.parent ? extract_const_path_text(node.parent) : nil
-          [parent, node.name.to_s].compact.join('::')
+          const_name = node.respond_to?(:name) ? node.name.to_s : node.child.name.to_s
+          [parent, const_name].compact.join('::')
         end
       end
 
       def extract_const_name(node)
-        case node
-        when Prism::ConstantReadNode
-          node.name.to_s
-        when Prism::ConstantPathNode
-          extract_const_path_text(node)
-        end
+        extract_const_path_text(node)
       end
 
       # ── Parser gem fallback ──────────────────────────────────────────────
@@ -490,7 +452,7 @@ module CodebaseIndex
           name_node = convert_parser_node(parser_node.children[0], source)
           superclass = parser_node.children[1] ? convert_parser_node(parser_node.children[1], source) : nil
           body = parser_node.children[2] ? convert_parser_node(parser_node.children[2], source) : nil
-          body_children = body&.type == :begin ? body.children : [body].compact
+          body_children = parser_body_children(body)
           children = [name_node, superclass] + body_children
           Node.new(
             type: :class,
@@ -502,7 +464,7 @@ module CodebaseIndex
         when :module
           name_node = convert_parser_node(parser_node.children[0], source)
           body = parser_node.children[1] ? convert_parser_node(parser_node.children[1], source) : nil
-          body_children = body&.type == :begin ? body.children : [body].compact
+          body_children = parser_body_children(body)
           children = [name_node] + body_children
           Node.new(
             type: :module,
@@ -513,7 +475,7 @@ module CodebaseIndex
           )
         when :def
           body = parser_node.children[2] ? convert_parser_node(parser_node.children[2], source) : nil
-          body_children = body&.type == :begin ? body.children : [body].compact
+          body_children = parser_body_children(body)
           Node.new(
             type: :def,
             children: body_children,
@@ -524,7 +486,7 @@ module CodebaseIndex
           )
         when :defs
           body = parser_node.children[3] ? convert_parser_node(parser_node.children[3], source) : nil
-          body_children = body&.type == :begin ? body.children : [body].compact
+          body_children = parser_body_children(body)
           receiver = parser_node.children[0].type == :self ? 'self' : parser_node.children[0].to_s
           Node.new(
             type: :defs,
@@ -605,13 +567,12 @@ module CodebaseIndex
         end
       end
 
-      def extract_parser_source_span(node, source)
-        lines = source.lines
-        start_idx = node.loc.line - 1
-        end_idx = node.loc.expression.last_line - 1
-        return nil if start_idx.negative? || end_idx >= lines.length
+      def parser_body_children(body_node)
+        body_node&.type == :begin ? body_node.children : [body_node].compact
+      end
 
-        lines[start_idx..end_idx].join
+      def extract_parser_source_span(node, source)
+        extract_source_span(source, node.loc.line, node.loc.expression.last_line)
       end
 
       def extract_parser_source_text(node, source)

data/lib/codebase_index/builder.rb

@@ -26,7 +26,7 @@ module CodebaseIndex
   #     config.vector_store_options = { url: ENV['QDRANT_URL'], collection: 'myapp' }
   #   end
   #
-  class Builder
+  class Builder # rubocop:disable Metrics/ClassLength
     # Named presets mapping to default adapter types.
     #
     # :local      — fully local, no external services required
@@ -74,14 +74,25 @@ module CodebaseIndex
 
     # Build a {Retriever} wired with adapters from the configuration.
     #
-    # @return [Retriever] A fully instantiated, wired retriever
+    # When `cache_enabled` is true, the embedding provider is wrapped with
+    # {Cache::CachedEmbeddingProvider} and the retriever is wrapped with
+    # {Cache::CachedRetriever} for transparent caching of expensive operations.
+    #
+    # @return [Retriever, Cache::CachedRetriever] A fully wired retriever
     def build_retriever
-      Retriever.new(
+      provider = build_embedding_provider
+      cache = build_cache_store
+
+      provider = wrap_with_embedding_cache(provider, cache) if cache
+
+      retriever = Retriever.new(
         vector_store: build_vector_store,
         metadata_store: build_metadata_store,
         graph_store: build_graph_store,
-        embedding_provider: build_embedding_provider
+        embedding_provider: provider
       )
+
+      cache ? wrap_with_retriever_cache(retriever, cache) : retriever
     end
 
     # Instantiate the vector store adapter specified by the configuration.
@@ -133,5 +144,57 @@ module CodebaseIndex
       else raise ArgumentError, "Unknown graph_store: #{@config.graph_store}"
       end
     end
+
+    # Build a cache store from configuration, or nil if caching is disabled.
+    #
+    # @return [Cache::CacheStore, nil]
+    def build_cache_store
+      return nil unless @config.cache_enabled
+
+      opts = @config.cache_options || {}
+
+      case @config.cache_store
+      when :memory
+        Cache::InMemory.new(max_entries: opts.fetch(:max_entries, 500))
+      when :redis
+        require_relative 'cache/redis_cache_store'
+        Cache::RedisCacheStore.new(redis: opts.fetch(:redis), default_ttl: opts[:default_ttl])
+      when :solid_cache
+        require_relative 'cache/solid_cache_store'
+        Cache::SolidCacheStore.new(cache: opts.fetch(:cache), default_ttl: opts[:default_ttl])
+      when Cache::CacheStore
+        @config.cache_store
+      else
+        raise ArgumentError, "Unknown cache_store: #{@config.cache_store}"
+      end
+    end
+
+    # Wrap an embedding provider with caching.
+    #
+    # @param provider [Embedding::Provider::Interface]
+    # @param cache [Cache::CacheStore]
+    # @return [Cache::CachedEmbeddingProvider]
+    def wrap_with_embedding_cache(provider, cache)
+      ttls = (@config.cache_options || {}).fetch(:ttl, {})
+      Cache::CachedEmbeddingProvider.new(
+        provider: provider,
+        cache_store: cache,
+        ttl: ttls.fetch(:embeddings, Cache::DEFAULT_TTLS[:embeddings])
+      )
+    end
+
+    # Wrap a retriever with caching.
+    #
+    # @param retriever [Retriever]
+    # @param cache [Cache::CacheStore]
+    # @return [Cache::CachedRetriever]
+    def wrap_with_retriever_cache(retriever, cache)
+      ttls = (@config.cache_options || {}).fetch(:ttl, {})
+      Cache::CachedRetriever.new(
+        retriever: retriever,
+        cache_store: cache,
+        context_ttl: ttls.fetch(:context, Cache::DEFAULT_TTLS[:context])
+      )
+    end
   end
 end

data/lib/codebase_index/cache/cache_middleware.rb

@@ -0,0 +1,199 @@
+# frozen_string_literal: true
+
+require 'digest'
+require_relative 'cache_store'
+
+module CodebaseIndex
+  module Cache
+    # Decorator that wraps an embedding provider with cache-through logic.
+    #
+    # Implements the same {Embedding::Provider::Interface} so it can be
+    # injected transparently in place of the real provider. On cache hit,
+    # the expensive API call (OpenAI, Ollama) is skipped entirely.
+    #
+    # @example
+    #   real_provider = Embedding::Provider::OpenAI.new(api_key: key)
+    #   cached = CachedEmbeddingProvider.new(provider: real_provider, cache_store: store)
+    #   cached.embed("How does User work?")  # API call + cache write
+    #   cached.embed("How does User work?")  # cache hit, no API call
+    #
+    class CachedEmbeddingProvider
+      include Embedding::Provider::Interface
+
+      # @param provider [Embedding::Provider::Interface] The real embedding provider
+      # @param cache_store [CacheStore] Cache backend instance
+      # @param ttl [Integer] TTL for cached embeddings in seconds
+      def initialize(provider:, cache_store:, ttl: DEFAULT_TTLS[:embeddings])
+        @provider = provider
+        @cache_store = cache_store
+        @ttl = ttl
+      end
+
+      # Embed a single text, returning a cached vector when available.
+      #
+      # @param text [String] Text to embed
+      # @return [Array<Float>] Embedding vector
+      def embed(text)
+        key = embedding_key(text)
+        @cache_store.fetch(key, ttl: @ttl) { @provider.embed(text) }
+      end
+
+      # Embed a batch of texts, using cached vectors for any previously seen texts.
+      #
+      # Only texts that are not already cached are sent to the real provider.
+      # Results are merged back in original order.
+      #
+      # @param texts [Array<String>] Texts to embed
+      # @return [Array<Array<Float>>] Embedding vectors (same order as input)
+      def embed_batch(texts)
+        results, misses, miss_indices = partition_cached(texts)
+
+        if misses.any?
+          fresh_vectors = @provider.embed_batch(misses)
+          misses.each_with_index do |text, i|
+            results[miss_indices[i]] = fresh_vectors[i]
+            begin
+              @cache_store.write(embedding_key(text), fresh_vectors[i], ttl: @ttl)
+            rescue StandardError => e
+              warn("[CodebaseIndex] CachedEmbeddingProvider cache write failed: #{e.message}")
+            end
+          end
+        end
+
+        results
+      end
+
+      # Delegate dimensions to the underlying provider.
+      #
+      # @return [Integer]
+      def dimensions
+        @provider.dimensions
+      end
+
+      # Delegate model_name to the underlying provider.
+      #
+      # @return [String]
+      def model_name
+        @provider.model_name
+      end
+
+      private
+
+      # Split texts into cached hits and uncached misses.
+      #
+      # @param texts [Array<String>]
+      # @return [Array(Array, Array<String>, Array<Integer>)]
+      def partition_cached(texts)
+        results = Array.new(texts.size)
+        misses = []
+        miss_indices = []
+
+        texts.each_with_index do |text, idx|
+          cached = @cache_store.read(embedding_key(text))
+          if cached
+            results[idx] = cached
+          else
+            misses << text
+            miss_indices << idx
+          end
+        end
+
+        [results, misses, miss_indices]
+      end
+
+      # Build a cache key for an embedding text.
+      #
+      # @param text [String]
+      # @return [String]
+      def embedding_key(text)
+        Cache.cache_key(:embeddings, Digest::SHA256.hexdigest(text))
+      end
+    end
+
+    # Decorator that wraps a {Retriever} with result caching.
+    #
+    # Caches the full formatted context output (the most token-expensive artifact)
+    # keyed by query + budget. Also caches the structural context overview
+    # separately with a longer TTL.
+    #
+    # @example
+    #   retriever = CodebaseIndex::Retriever.new(...)
+    #   cached = CachedRetriever.new(retriever: retriever, cache_store: store)
+    #   cached.retrieve("How does User work?")  # full pipeline + cache
+    #   cached.retrieve("How does User work?")  # instant cache hit
+    #
+    class CachedRetriever
+      # @param retriever [Retriever] The real retriever instance
+      # @param cache_store [CacheStore] Cache backend instance
+      # @param context_ttl [Integer] TTL for formatted context results
+      def initialize(retriever:, cache_store:, context_ttl: DEFAULT_TTLS[:context])
+        @retriever = retriever
+        @cache_store = cache_store
+        @context_ttl = context_ttl
+      end
+
+      # Execute the retrieval pipeline with context-level caching.
+      #
+      # On cache hit, returns a RetrievalResult reconstructed from cached data
+      # without running any pipeline stages. On miss, delegates to the real
+      # retriever and caches the serializable parts of the result.
+      #
+      # @param query [String] Natural language query
+      # @param budget [Integer] Token budget
+      # @return [Retriever::RetrievalResult]
+      def retrieve(query, budget: 8000)
+        key = context_key(query, budget)
+        cached = @cache_store.read(key)
+
+        if cached
+          return Retriever::RetrievalResult.new(
+            context: cached['context'],
+            sources: cached['sources'],
+            classification: nil,
+            strategy: cached['strategy']&.to_sym,
+            tokens_used: cached['tokens_used'],
+            budget: budget,
+            trace: nil
+          )
+        end
+
+        result = @retriever.retrieve(query, budget: budget)
+
+        begin
+          @cache_store.write(key, serialize_result(result), ttl: @context_ttl)
+        rescue StandardError => e
+          warn("[CodebaseIndex] CachedRetriever cache write failed: #{e.message}")
+        end
+
+        result
+      end
+
+      private
+
+      # Build a cache key for a context result.
+      #
+      # @param query [String]
+      # @param budget [Integer]
+      # @return [String]
+      def context_key(query, budget)
+        Cache.cache_key(:context, query, budget.to_s)
+      end
+
+      # Serialize a RetrievalResult to a JSON-safe hash.
+      #
+      # Only caches the fields needed to reconstruct a useful result:
+      # context string, sources list, strategy, and token count.
+      #
+      # @param result [Retriever::RetrievalResult]
+      # @return [Hash]
+      def serialize_result(result)
+        {
+          'context' => result.context,
+          'sources' => result.sources,
+          'strategy' => result.strategy&.to_s,
+          'tokens_used' => result.tokens_used
+        }
+      end
+    end
+  end
+end