codebase_index 0.2.1 → 0.3.1

data/lib/codebase_index/console/adapters/job_adapter.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module CodebaseIndex
+  module Console
+    module Adapters
+      # Base class for job backend adapters.
+      #
+      # Subclasses implement `self.available?` and a private `prefix` method.
+      # The prefix is used to build bridge tool names (e.g., "sidekiq_queue_stats").
+      #
+      # @example
+      #   class MyAdapter < JobAdapter
+      #     def self.available? = !!defined?(::MyQueue)
+      #     private
+      #     def prefix = 'my_queue'
+      #   end
+      #
+      class JobAdapter
+        # Get queue statistics (sizes, latencies).
+        #
+        # @return [Hash] Bridge request
+        def queue_stats
+          { tool: "#{prefix}_queue_stats", params: {} }
+        end
+
+        # List recent job failures.
+        #
+        # @param limit [Integer] Max failures (default: 10, max: 100)
+        # @return [Hash] Bridge request
+        def recent_failures(limit: 10)
+          limit = [limit, 100].min
+          { tool: "#{prefix}_recent_failures", params: { limit: limit } }
+        end
+
+        # Find a job by its ID.
+        #
+        # @param id [Object] Job ID
+        # @return [Hash] Bridge request
+        def find_job(id:)
+          { tool: "#{prefix}_find_job", params: { id: id } }
+        end
+
+        # List scheduled jobs.
+        #
+        # @param limit [Integer] Max jobs (default: 20, max: 100)
+        # @return [Hash] Bridge request
+        def scheduled_jobs(limit: 20)
+          limit = [limit, 100].min
+          { tool: "#{prefix}_scheduled_jobs", params: { limit: limit } }
+        end
+
+        # Retry a failed job.
+        #
+        # @param id [Object] Job ID
+        # @return [Hash] Bridge request
+        def retry_job(id:)
+          { tool: "#{prefix}_retry_job", params: { id: id } }
+        end
+
+        private
+
+        def prefix
+          raise NotImplementedError, "#{self.class}#prefix must be implemented"
+        end
+      end
+    end
+  end
+end

data/lib/codebase_index/console/adapters/sidekiq_adapter.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative 'job_adapter'
+
 module CodebaseIndex
   module Console
     module Adapters
@@ -12,53 +14,18 @@ module CodebaseIndex
       #   adapter = SidekiqAdapter.new
       #   adapter.queue_stats  # => { tool: 'sidekiq_queue_stats', params: {} }
       #
-      class SidekiqAdapter
+      class SidekiqAdapter < JobAdapter
         # Check if Sidekiq is available in the current environment.
         #
         # @return [Boolean]
         def self.available?
-          defined?(::Sidekiq) ? true : false
+          !!defined?(::Sidekiq)
         end
 
-        # Get queue statistics (sizes, latencies).
-        #
-        # @return [Hash] Bridge request
-        def queue_stats
-          { tool: 'sidekiq_queue_stats', params: {} }
-        end
+        private
 
-        # List recent job failures.
-        #
-        # @param limit [Integer] Max failures (default: 10, max: 100)
-        # @return [Hash] Bridge request
-        def recent_failures(limit: 10)
-          limit = [limit, 100].min
-          { tool: 'sidekiq_recent_failures', params: { limit: limit } }
-        end
-
-        # Find a job by its ID.
-        #
-        # @param id [String] Sidekiq job ID
-        # @return [Hash] Bridge request
-        def find_job(id:)
-          { tool: 'sidekiq_find_job', params: { id: id } }
-        end
-
-        # List scheduled jobs.
-        #
-        # @param limit [Integer] Max jobs (default: 20, max: 100)
-        # @return [Hash] Bridge request
-        def scheduled_jobs(limit: 20)
-          limit = [limit, 100].min
-          { tool: 'sidekiq_scheduled_jobs', params: { limit: limit } }
-        end
-
-        # Retry a failed job.
-        #
-        # @param id [String] Sidekiq job ID
-        # @return [Hash] Bridge request
-        def retry_job(id:)
-          { tool: 'sidekiq_retry_job', params: { id: id } }
+        def prefix
+          'sidekiq'
         end
       end
     end

data/lib/codebase_index/console/adapters/solid_queue_adapter.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative 'job_adapter'
+
 module CodebaseIndex
   module Console
     module Adapters
@@ -12,53 +14,18 @@ module CodebaseIndex
       #   adapter = SolidQueueAdapter.new
       #   adapter.queue_stats  # => { tool: 'solid_queue_queue_stats', params: {} }
       #
-      class SolidQueueAdapter
+      class SolidQueueAdapter < JobAdapter
         # Check if Solid Queue is available in the current environment.
         #
         # @return [Boolean]
         def self.available?
-          defined?(::SolidQueue) ? true : false
+          !!defined?(::SolidQueue)
         end
 
-        # Get queue statistics (sizes, latencies).
-        #
-        # @return [Hash] Bridge request
-        def queue_stats
-          { tool: 'solid_queue_queue_stats', params: {} }
-        end
+        private
 
-        # List recent job failures.
-        #
-        # @param limit [Integer] Max failures (default: 10, max: 100)
-        # @return [Hash] Bridge request
-        def recent_failures(limit: 10)
-          limit = [limit, 100].min
-          { tool: 'solid_queue_recent_failures', params: { limit: limit } }
-        end
-
-        # Find a job by its ID.
-        #
-        # @param id [Object] Solid Queue job ID
-        # @return [Hash] Bridge request
-        def find_job(id:)
-          { tool: 'solid_queue_find_job', params: { id: id } }
-        end
-
-        # List scheduled jobs.
-        #
-        # @param limit [Integer] Max jobs (default: 20, max: 100)
-        # @return [Hash] Bridge request
-        def scheduled_jobs(limit: 20)
-          limit = [limit, 100].min
-          { tool: 'solid_queue_scheduled_jobs', params: { limit: limit } }
-        end
-
-        # Retry a failed job.
-        #
-        # @param id [Object] Solid Queue job ID
-        # @return [Hash] Bridge request
-        def retry_job(id:)
-          { tool: 'solid_queue_retry_job', params: { id: id } }
+        def prefix
+          'solid_queue'
         end
       end
     end

data/lib/codebase_index/console/bridge.rb

@@ -23,6 +23,8 @@ module CodebaseIndex
     #
     class Bridge
       SUPPORTED_TOOLS = %w[count sample find pluck aggregate association_count schema recent status].freeze
+      # Alias used by EmbeddedExecutor to avoid duplicating the list.
+      TIER1_TOOLS = SUPPORTED_TOOLS
       TOOL_HANDLERS = SUPPORTED_TOOLS.to_h { |t| [t, :"handle_#{t}"] }.freeze
 
       # @param input [IO] Input stream (reads JSON-lines)
@@ -113,6 +115,11 @@ module CodebaseIndex
         @model_validator.validate_model!(model)
       end
 
+      # Stub handlers below return empty/zero data by design.
+      # This Bridge class is a protocol scaffold — real execution happens
+      # in EmbeddedExecutor (in-process) or a live Rails bridge process.
+      # The stubs satisfy the protocol contract for testing and offline use.
+
       def handle_count(_params)
         { 'count' => 0 }
       end

data/lib/codebase_index/console/console_response_renderer.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative '../mcp/tool_response_renderer'
+require_relative '../mcp/renderers/json_renderer'
 
 module CodebaseIndex
   module Console
@@ -66,13 +67,8 @@ module CodebaseIndex
     end
 
     # JSON passthrough renderer for backward compatibility.
-    # Returns JSON.pretty_generate output for any data.
-    class JsonConsoleRenderer < MCP::ToolResponseRenderer
-      # @param data [Object] Any JSON-serializable data
-      # @return [String] Pretty-printed JSON
-      def render_default(data)
-        JSON.pretty_generate(data)
-      end
+    # Delegates to MCP::Renderers::JsonRenderer for consistent JSON output.
+    class JsonConsoleRenderer < MCP::Renderers::JsonRenderer
     end
   end
 end

data/lib/codebase_index/console/embedded_executor.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative 'bridge'
 require_relative 'model_validator'
 require_relative 'safe_context'
 
@@ -20,7 +21,7 @@ module CodebaseIndex
     class EmbeddedExecutor # rubocop:disable Metrics/ClassLength
       AGGREGATE_FUNCTIONS = %w[sum average minimum maximum].freeze
 
-      TIER1_TOOLS = %w[count sample find pluck aggregate association_count schema recent status].freeze
+      TIER1_TOOLS = Bridge::TIER1_TOOLS
 
       # @param model_validator [ModelValidator] Validates model/column names
       # @param safe_context [SafeContext] Wraps execution in rolled-back transaction

data/lib/codebase_index/console/server.rb

@@ -567,15 +567,12 @@ module CodebaseIndex
         # rubocop:disable Metrics/ParameterLists
         def define_console_tool(server, conn_mgr, name, description, properties:, required: nil,
                                 safe_ctx: nil, renderer: nil, &tool_block)
-          mgr = conn_mgr
-          ctx = safe_ctx
-          rdr = renderer
           bridge_method = method(:send_to_bridge)
           schema = { properties: properties }
           schema[:required] = required if required&.any?
           server.define_tool(name: name, description: description, input_schema: schema) do |server_context:, **args|
             request = tool_block.call(args)
-            bridge_method.call(mgr, request.transform_keys(&:to_s), ctx, renderer: rdr)
+            bridge_method.call(conn_mgr, request.transform_keys(&:to_s), safe_ctx, renderer: renderer)
           end
         end
         # rubocop:enable Metrics/ParameterLists

data/lib/codebase_index/dependency_graph.rb

@@ -24,15 +24,18 @@ module CodebaseIndex
     def initialize
       @nodes = {}      # identifier => { type:, file_path: }
       @edges = {}      # identifier => [dependency identifiers]
-      @reverse = {}    # identifier => [dependent identifiers]
+      @reverse = {}    # identifier => Set of dependent identifiers
       @file_map = {}   # file_path => identifier
-      @type_index = {} # type => [identifiers]
+      @type_index = {} # type => Set of identifiers
+      @to_h = nil
     end
 
     # Register a unit in the graph
     #
     # @param unit [ExtractedUnit] The unit to register
     def register(unit)
+      @to_h = nil
+
       @nodes[unit.identifier] = {
         type: unit.type,
         file_path: unit.file_path,
@@ -42,14 +45,12 @@ module CodebaseIndex
       @edges[unit.identifier] = unit.dependencies.map { |d| d[:target] }
       @file_map[unit.file_path] = unit.identifier if unit.file_path
 
-      # Type index for filtering
-      @type_index[unit.type] ||= []
-      @type_index[unit.type] << unit.identifier unless @type_index[unit.type].include?(unit.identifier)
+      # Type index for filtering (Set-based for O(1) insert)
+      (@type_index[unit.type] ||= Set.new).add(unit.identifier)
 
-      # Build reverse edges
+      # Build reverse edges (Set-based for O(1) insert)
       unit.dependencies.each do |dep|
-        @reverse[dep[:target]] ||= []
-        @reverse[dep[:target]] << unit.identifier unless @reverse[dep[:target]].include?(unit.identifier)
+        (@reverse[dep[:target]] ||= Set.new).add(unit.identifier)
       end
     end
 
@@ -116,7 +117,7 @@ module CodebaseIndex
     # @param identifier [String] Unit identifier
     # @return [Array<String>] List of dependent identifiers
     def dependents_of(identifier)
-      @reverse[identifier] || []
+      @reverse.fetch(identifier, Set.new).to_a
     end
 
     # Get all units of a specific type
@@ -124,7 +125,7 @@ module CodebaseIndex
     # @param type [Symbol] Unit type (:model, :controller, etc.)
     # @return [Array<String>] List of unit identifiers
     def units_of_type(type)
-      @type_index[type] || []
+      @type_index.fetch(type, Set.new).to_a
     end
 
     # Compute PageRank scores for all nodes
@@ -140,18 +141,19 @@ module CodebaseIndex
       n = @nodes.size
       return {} if n.zero?
 
+      node_ids = @nodes.keys
       base_score = 1.0 / n
-      scores = @nodes.keys.to_h { |id| [id, base_score] }
+      scores = node_ids.to_h { |id| [id, base_score] }
 
       iterations.times do
         # Collect rank from dangling nodes (no outgoing edges) and redistribute
-        dangling_sum = @nodes.keys.sum do |id|
+        dangling_sum = node_ids.sum do |id|
           @edges[id].nil? || @edges[id].empty? ? scores[id] : 0.0
         end
 
         new_scores = {}
 
-        @nodes.each_key do |id|
+        node_ids.each do |id|
           # Sum contributions from nodes that depend on this one
           incoming = @reverse[id] || []
           rank_sum = incoming.sum do |src|
@@ -168,22 +170,24 @@ module CodebaseIndex
       scores
     end
 
-    # Serialize graph for persistence
+    # Serialize graph for persistence. Memoized — cache is invalidated on register.
+    # Returns a dup so callers can't pollute the cached hash.
     #
     # @return [Hash] Complete graph data
     def to_h
-      {
+      @to_h ||= {
         nodes: @nodes,
         edges: @edges,
-        reverse: @reverse,
+        reverse: @reverse.transform_values(&:to_a),
         file_map: @file_map,
-        type_index: @type_index,
+        type_index: @type_index.transform_values(&:to_a),
         stats: {
           node_count: @nodes.size,
           edge_count: @edges.values.sum(&:size),
           types: @type_index.transform_values(&:size)
         }
       }
+      @to_h.dup
     end
 
     # Load graph from persisted data
@@ -201,11 +205,16 @@ module CodebaseIndex
       graph.instance_variable_set(:@nodes, raw_nodes.transform_values { |v| symbolize_node(v) })
 
       graph.instance_variable_set(:@edges, data[:edges] || data['edges'] || {})
-      graph.instance_variable_set(:@reverse, data[:reverse] || data['reverse'] || {})
+
+      raw_reverse = data[:reverse] || data['reverse'] || {}
+      graph.instance_variable_set(:@reverse, raw_reverse.transform_values { |v| v.is_a?(Set) ? v : Set.new(v) })
+
       graph.instance_variable_set(:@file_map, data[:file_map] || data['file_map'] || {})
 
       raw_type_index = data[:type_index] || data['type_index'] || {}
-      graph.instance_variable_set(:@type_index, raw_type_index.transform_keys(&:to_sym))
+      graph.instance_variable_set(:@type_index, raw_type_index.transform_keys(&:to_sym).transform_values do |v|
+        v.is_a?(Set) ? v : Set.new(v)
+      end)
 
       graph
     end

data/lib/codebase_index/embedding/indexer.rb

@@ -9,12 +9,14 @@ module CodebaseIndex
     # generates embeddings, and stores vectors. Supports full and incremental
     # modes with checkpoint-based resumability.
     class Indexer
-      def initialize(provider:, text_preparer:, vector_store:, output_dir:, batch_size: 32)
+      # @param checkpoint_interval [Integer] Save checkpoint every N batches (default: 10)
+      def initialize(provider:, text_preparer:, vector_store:, output_dir:, batch_size: 32, checkpoint_interval: 10) # rubocop:disable Metrics/ParameterLists
         @provider = provider
         @text_preparer = text_preparer
         @vector_store = vector_store
         @output_dir = output_dir
         @batch_size = batch_size
+        @checkpoint_interval = checkpoint_interval
       end
 
       # Index all extracted units (full mode). Returns stats hash.
@@ -44,12 +46,17 @@ module CodebaseIndex
       def process_units(units, incremental:)
         checkpoint = incremental ? load_checkpoint : {}
         stats = { processed: 0, skipped: 0, errors: 0 }
+        batch_count = 0
 
         units.each_slice(@batch_size) do |batch|
           process_batch(batch, checkpoint, stats, incremental: incremental)
-          save_checkpoint(checkpoint)
+          batch_count += 1
+          save_checkpoint(checkpoint) if (batch_count % @checkpoint_interval).zero?
         end
 
+        # Always save final checkpoint
+        save_checkpoint(checkpoint)
+
         stats
       end
 
@@ -98,16 +105,19 @@ module CodebaseIndex
         store_vectors(items, vectors, checkpoint, stats)
       rescue StandardError => e
         stats[:errors] += items.size
-        stats[:error_messages] ||= []
-        stats[:error_messages] << e.message
         raise CodebaseIndex::Error, "Embedding failed: #{e.message}"
       end
 
       def store_vectors(items, vectors, checkpoint, stats)
-        items.each_with_index do |item, idx|
-          metadata = { type: item[:unit_data]['type'], identifier: item[:identifier],
-                       file_path: item[:unit_data]['file_path'] }
-          @vector_store.store(item[:id], vectors[idx], metadata)
+        entries = items.each_with_index.map do |item, idx|
+          { id: item[:id], vector: vectors[idx],
+            metadata: { type: item[:unit_data]['type'], identifier: item[:identifier],
+                        file_path: item[:unit_data]['file_path'] } }
+        end
+
+        @vector_store.store_batch(entries)
+
+        items.each do |item|
           checkpoint[item[:identifier]] = item[:source_hash]
           stats[:processed] += 1
         end

data/lib/codebase_index/embedding/openai.rb

@@ -81,24 +81,45 @@ module CodebaseIndex
         # @return [Hash] parsed JSON response
         # @raise [CodebaseIndex::Error] if the API returns a non-success status
         def post_request(body)
-          http = Net::HTTP.new(ENDPOINT.host, ENDPOINT.port)
-          http.use_ssl = true
-          http.open_timeout = 10
-          http.read_timeout = 30
-
           request = Net::HTTP::Post.new(ENDPOINT.path)
           request['Content-Type'] = 'application/json'
           request['Authorization'] = "Bearer #{@api_key}"
           request.body = body.to_json
 
-          response = http.request(request)
+          response = http_client.request(request)
+
+          unless response.is_a?(Net::HTTPSuccess)
+            raise CodebaseIndex::Error, "OpenAI API error: #{response.code} #{response.body}"
+          end
 
+          JSON.parse(response.body)
+        rescue Errno::ECONNRESET, Net::OpenTimeout, IOError
+          # Connection dropped — reset and retry once
+          @http_client = nil
+          response = http_client.request(request)
           unless response.is_a?(Net::HTTPSuccess)
             raise CodebaseIndex::Error, "OpenAI API error: #{response.code} #{response.body}"
           end
 
           JSON.parse(response.body)
         end
+
+        # Return a reusable, started HTTP client for the OpenAI API.
+        # Calling http.start opens a persistent TCP connection so
+        # keep_alive_timeout actually takes effect across requests.
+        #
+        # @return [Net::HTTP]
+        def http_client
+          return @http_client if @http_client&.started?
+
+          http = Net::HTTP.new(ENDPOINT.host, ENDPOINT.port)
+          http.use_ssl = true
+          http.open_timeout = 10
+          http.read_timeout = 30
+          http.keep_alive_timeout = 30
+          http.start
+          @http_client = http
+        end
       end
     end
   end

data/lib/codebase_index/embedding/provider.rb

@@ -118,17 +118,44 @@ module CodebaseIndex
         # @return [Hash] parsed JSON response
         # @raise [CodebaseIndex::Error] if the API returns a non-success status
         def post_request(body)
-          http = Net::HTTP.new(@uri.host, @uri.port)
           request = Net::HTTP::Post.new(@uri.path, 'Content-Type' => 'application/json')
           request.body = body.to_json
-          response = http.request(request)
+          response = http_client.request(request)
+
+          unless response.is_a?(Net::HTTPSuccess)
+            raise CodebaseIndex::Error, "Ollama API error: #{response.code} #{response.body}"
+          end
 
+          JSON.parse(response.body)
+        rescue Errno::ECONNRESET, Net::OpenTimeout, Net::ReadTimeout, IOError
+          # Connection dropped — reset and retry once
+          @http_client = nil
+          begin
+            response = http_client.request(request)
+          rescue StandardError => retry_error
+            raise CodebaseIndex::Error, "Ollama API error (retry failed): #{retry_error.message}"
+          end
           unless response.is_a?(Net::HTTPSuccess)
             raise CodebaseIndex::Error, "Ollama API error: #{response.code} #{response.body}"
           end
 
           JSON.parse(response.body)
         end
+
+        # Return a reusable, started HTTP client for the Ollama API.
+        #
+        # @return [Net::HTTP]
+        def http_client
+          return @http_client if @http_client&.started?
+
+          http = Net::HTTP.new(@uri.host, @uri.port)
+          http.use_ssl = @uri.scheme == 'https'
+          http.open_timeout = 10
+          http.read_timeout = 30
+          http.keep_alive_timeout = 30
+          http.start
+          @http_client = http
+        end
       end
     end
   end

data/lib/codebase_index/evaluation/evaluator.rb

@@ -23,6 +23,8 @@ module CodebaseIndex
       # Aggregate report across all queries.
       EvaluationReport = Struct.new(:results, :aggregates, keyword_init: true)
 
+      METRIC_KEYS = %i[precision_at5 precision_at10 recall mrr context_completeness token_efficiency].freeze
+
       # @param retriever [CodebaseIndex::Retriever] Configured retriever instance
       # @param query_set [QuerySet] Set of evaluation queries with ground truth
       # @param budget [Integer] Token budget per query
@@ -113,10 +115,9 @@ module CodebaseIndex
       def compute_aggregates(results)
         return empty_aggregates if results.empty?
 
-        metric_keys = %i[precision_at5 precision_at10 recall mrr context_completeness token_efficiency]
         aggregates = {}
 
-        metric_keys.each do |key|
+        METRIC_KEYS.each do |key|
           values = results.map { |r| r.scores[key] }
           aggregates[:"mean_#{key}"] = values.sum / values.size.to_f
         end
@@ -130,16 +131,8 @@ module CodebaseIndex
       #
       # @return [Hash]
       def empty_aggregates
-        {
-          mean_precision_at5: 0.0,
-          mean_precision_at10: 0.0,
-          mean_recall: 0.0,
-          mean_mrr: 0.0,
-          mean_context_completeness: 0.0,
-          mean_token_efficiency: 0.0,
-          total_queries: 0,
-          mean_tokens_used: 0.0
-        }
+        METRIC_KEYS.to_h { |key| [:"mean_#{key}", 0.0] }
+                   .merge(total_queries: 0, mean_tokens_used: 0.0)
       end
     end
   end