codebase_index 0.2.1 → 0.3.1

data/lib/codebase_index/storage/metadata_store.rb

@@ -35,6 +35,20 @@ module CodebaseIndex
           raise NotImplementedError
         end
 
+        # Find multiple units by IDs in a single query.
+        #
+        # Default implementation falls back to individual find calls.
+        # Adapters should override for batch-optimized behavior.
+        #
+        # @param ids [Array<String>] The identifiers to look up
+        # @return [Hash<String, Hash>] Map of id => metadata for found units
+        def find_batch(ids)
+          ids.each_with_object({}) do |id, result|
+            data = find(id)
+            result[id] = data if data
+          end
+        end
+
         # Find all units of a given type.
         #
         # @param type [String] The unit type to filter by
@@ -112,14 +126,21 @@ module CodebaseIndex
           JSON.parse(row['data'])
         end
 
+        # @see Interface#find_batch
+        def find_batch(ids)
+          return {} if ids.empty?
+
+          placeholders = Array.new(ids.size, '?').join(', ')
+          rows = @db.execute("SELECT id, data FROM units WHERE id IN (#{placeholders})", ids)
+          rows.to_h do |row|
+            [row['id'], JSON.parse(row['data'])]
+          end
+        end
+
         # @see Interface#find_by_type
         def find_by_type(type)
           rows = @db.execute('SELECT id, data FROM units WHERE type = ?', [type.to_s])
-          rows.map do |row|
-            parsed = JSON.parse(row['data'])
-            parsed['id'] = row['id']
-            parsed
-          end
+          rows.map { |row| parse_row(row) }
         end
 
         # @see Interface#search
@@ -132,11 +153,7 @@ module CodebaseIndex
             rows = @db.execute('SELECT id, data FROM units WHERE data LIKE ?', ["%#{query}%"])
           end
 
-          rows.map do |row|
-            parsed = JSON.parse(row['data'])
-            parsed['id'] = row['id']
-            parsed
-          end
+          rows.map { |row| parse_row(row) }
         end
 
         # @see Interface#delete
@@ -151,6 +168,16 @@ module CodebaseIndex
 
         private
 
+        # Parse a database row into a metadata hash with the id field injected.
+        #
+        # @param row [Hash] Database row with 'id' and 'data' keys
+        # @return [Hash] Parsed metadata with 'id' key set
+        def parse_row(row)
+          parsed = JSON.parse(row['data'])
+          parsed['id'] = row['id']
+          parsed
+        end
+
         # Create the units table if it doesn't exist.
         def create_table
           @db.execute(<<~SQL)

data/lib/codebase_index/storage/pgvector.rb

@@ -17,7 +17,7 @@ module CodebaseIndex
       #   store.store("User", [0.1, 0.2, ...], { type: "model" })
       #   results = store.search([0.1, 0.2, ...], limit: 5, filters: { type: "model" })
       #
-      class Pgvector
+      class Pgvector # rubocop:disable Metrics/ClassLength
         include Interface
 
         TABLE = 'codebase_index_vectors'
@@ -56,13 +56,32 @@ module CodebaseIndex
         # @see Interface#store
         def store(id, vector, metadata = {})
           validate_vector!(vector)
-          quoted_id = @connection.quote(id)
-          quoted_metadata = @connection.quote(JSON.generate(metadata))
-          vector_literal = "[#{vector.join(',')}]"
+          entry = format_entry(id, vector, metadata)
 
           @connection.execute(<<~SQL)
             INSERT INTO #{TABLE} (id, embedding, metadata, created_at)
-            VALUES (#{quoted_id}, '#{vector_literal}', #{quoted_metadata}::jsonb, CURRENT_TIMESTAMP)
+            VALUES #{entry}
+            ON CONFLICT (id) DO UPDATE SET
+              embedding = EXCLUDED.embedding,
+              metadata = EXCLUDED.metadata,
+              created_at = CURRENT_TIMESTAMP
+          SQL
+        end
+
+        # Store multiple vectors in a single multi-row INSERT.
+        #
+        # @param entries [Array<Hash>] Each entry has :id, :vector, :metadata keys
+        def store_batch(entries)
+          return if entries.empty?
+
+          values = entries.map do |entry|
+            validate_vector!(entry[:vector])
+            format_entry(entry[:id], entry[:vector], entry[:metadata] || {})
+          end
+
+          @connection.execute(<<~SQL)
+            INSERT INTO #{TABLE} (id, embedding, metadata, created_at)
+            VALUES #{values.join(",\n")}
             ON CONFLICT (id) DO UPDATE SET
               embedding = EXCLUDED.embedding,
               metadata = EXCLUDED.metadata,
@@ -114,6 +133,19 @@ module CodebaseIndex
 
         private
 
+        # Format a single entry as a SQL VALUES tuple.
+        #
+        # @param id [String] Unique identifier
+        # @param vector [Array<Float>] Embedding vector
+        # @param metadata [Hash] Entry metadata
+        # @return [String] SQL values row literal
+        def format_entry(id, vector, metadata)
+          quoted_id = @connection.quote(id)
+          quoted_metadata = @connection.quote(JSON.generate(metadata))
+          vector_literal = "[#{vector.join(',')}]"
+          "(#{quoted_id}, '#{vector_literal}', #{quoted_metadata}::jsonb, CURRENT_TIMESTAMP)"
+        end
+
         # Convert a database row to a SearchResult.
         #
         # @param row [Hash] Database row with id, distance, metadata

data/lib/codebase_index/storage/qdrant.rb

@@ -19,7 +19,7 @@ module CodebaseIndex
       #   store.store("User", [0.1, 0.2, ...], { type: "model" })
       #   results = store.search([0.1, 0.2, ...], limit: 5)
       #
-      class Qdrant
+      class Qdrant # rubocop:disable Metrics/ClassLength
         include Interface
 
         # @param url [String] Qdrant server URL
@@ -64,6 +64,25 @@ module CodebaseIndex
           request(:put, "/collections/#{@collection}/points", body)
         end
 
+        # Store multiple vectors in a single batch upsert request.
+        #
+        # Sends the entire entries array in one HTTP call. Callers are responsible
+        # for chunking into reasonable batch sizes (e.g., 100–500 points) before
+        # calling this method; the embedding Indexer's +batch_size+ config controls
+        # the upstream chunk size.
+        #
+        # @param entries [Array<Hash>] Each entry has :id, :vector, :metadata keys
+        def store_batch(entries)
+          return if entries.empty?
+
+          body = {
+            points: entries.map do |entry|
+              { id: entry[:id], vector: entry[:vector], payload: entry[:metadata] || {} }
+            end
+          }
+          request(:put, "/collections/#{@collection}/points", body)
+        end
+
         # Search for similar vectors.
         #
         # @param query_vector [Array<Float>] The query embedding
@@ -130,26 +149,40 @@ module CodebaseIndex
         # @return [Hash] Parsed JSON response
         # @raise [CodebaseIndex::Error] if the API returns a non-success status
         def request(method, path, body = nil)
-          http = build_http
           req = build_request(method, path, body)
-          response = http.request(req)
+          response = http_client.request(req)
 
           unless response.is_a?(Net::HTTPSuccess)
             raise CodebaseIndex::Error, "Qdrant API error: #{response.code} #{response.body}"
           end
 
+          JSON.parse(response.body)
+        rescue Errno::ECONNRESET, Net::OpenTimeout, IOError
+          # Connection dropped — reset and retry once
+          @http_client = nil
+          response = http_client.request(req)
+          unless response.is_a?(Net::HTTPSuccess)
+            raise CodebaseIndex::Error, "Qdrant API error: #{response.code} #{response.body}"
+          end
+
           JSON.parse(response.body)
         end
 
-        # Build an HTTP client for the Qdrant server.
+        # Return a reusable, started HTTP client for the Qdrant server.
+        # Calling http.start opens a persistent TCP connection so
+        # keep_alive_timeout actually takes effect across requests.
         #
         # @return [Net::HTTP]
-        def build_http
+        def http_client
+          return @http_client if @http_client&.started?
+
           http = Net::HTTP.new(@uri.host, @uri.port)
           http.use_ssl = @uri.scheme == 'https'
           http.open_timeout = 10
           http.read_timeout = 30
-          http
+          http.keep_alive_timeout = 30
+          http.start
+          @http_client = http
         end
 
         # Build an HTTP request with headers and body.

data/lib/codebase_index/storage/vector_store.rb

@@ -25,6 +25,17 @@ module CodebaseIndex
           raise NotImplementedError
         end
 
+        # Store multiple vectors in a single batch operation.
+        #
+        # Default implementation falls back to individual store calls.
+        # Adapters should override for bulk-optimized behavior (e.g.,
+        # multi-row INSERT for pgvector, batch upsert for Qdrant).
+        #
+        # @param entries [Array<Hash>] Each entry has :id, :vector, :metadata keys
+        def store_batch(entries)
+          entries.each { |e| store(e[:id], e[:vector], e[:metadata] || {}) }
+        end
+
         # Search for similar vectors using cosine similarity.
         #
         # @param query_vector [Array<Float>] The query embedding vector

data/lib/codebase_index/temporal/snapshot_store.rb

@@ -230,16 +230,20 @@ module CodebaseIndex
           VALUES (?, ?, ?, ?, ?, ?)
         SQL
 
-        unit_hashes.each do |uh|
-          params = [
-            snapshot_id,
-            uh[:identifier] || uh['identifier'],
-            (uh[:type] || uh['type']).to_s,
-            uh[:source_hash] || uh['source_hash'],
-            uh[:metadata_hash] || uh['metadata_hash'],
-            uh[:dependencies_hash] || uh['dependencies_hash']
-          ]
-          @db.execute(sql, params)
+        # Wrap in a transaction to batch all inserts into a single commit,
+        # reducing per-row fsync overhead from O(n) to O(1).
+        @db.transaction do
+          unit_hashes.each do |uh|
+            params = [
+              snapshot_id,
+              mget(uh, 'identifier'),
+              mget(uh, 'type').to_s,
+              mget(uh, 'source_hash'),
+              mget(uh, 'metadata_hash'),
+              mget(uh, 'dependencies_hash')
+            ]
+            @db.execute(sql, params)
+          end
         end
       end
 

data/lib/codebase_index/token_utils.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module CodebaseIndex
+  # Shared token estimation utility.
+  #
+  # Uses project convention: (string.length / 4.0).ceil
+  # See docs/TOKEN_BENCHMARK.md — conservative floor (~10.6% overestimate).
+  module TokenUtils
+    module_function
+
+    # Estimate token count for a string.
+    #
+    # @param text [String] Text to estimate
+    # @return [Integer] Estimated token count
+    def estimate_tokens(text)
+      (text.length / 4.0).ceil
+    end
+  end
+end

data/lib/codebase_index/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module CodebaseIndex
-  VERSION = '0.2.1'
+  VERSION = '0.3.1'
 end

data/lib/codebase_index.rb

@@ -42,8 +42,10 @@ module CodebaseIndex
                   :concurrent_extraction, :precompute_flows, :enable_snapshots,
                   :session_tracer_enabled, :session_store, :session_id_proc, :session_exclude_paths,
                   :console_mcp_enabled, :console_mcp_path, :console_redacted_columns,
-                  :notion_api_token, :notion_database_ids
-    attr_reader :max_context_tokens, :similarity_threshold, :extractors, :pretty_json, :context_format
+                  :notion_api_token, :notion_database_ids,
+                  :cache_store, :cache_options
+    attr_reader :max_context_tokens, :similarity_threshold, :extractors, :pretty_json, :context_format,
+                :cache_enabled
 
     def initialize # rubocop:disable Metrics/MethodLength
       @output_dir = nil # Resolved lazily; Rails.root is nil at require time
@@ -68,6 +70,9 @@ module CodebaseIndex
       @console_redacted_columns = []
       @notion_api_token = nil
       @notion_database_ids = {}
+      @cache_enabled = false
+      @cache_store = nil      # :redis, :solid_cache, :memory, or a CacheStore instance
+      @cache_options = {}     # { redis: client, cache: store, ttl: { embeddings: 86400, ... } }
     end
 
     # @return [Pathname, String] Output directory, defaulting to Rails.root/tmp/codebase_index
@@ -119,10 +124,7 @@ module CodebaseIndex
     # @param value [Boolean] Must be true or false
     # @raise [ConfigurationError] if value is not a boolean
     def pretty_json=(value)
-      unless value.is_a?(TrueClass) || value.is_a?(FalseClass)
-        raise ConfigurationError, "pretty_json must be true or false, got #{value.inspect}"
-      end
-
+      validate_boolean!(:pretty_json, value)
       @pretty_json = value
     end
 
@@ -137,6 +139,13 @@ module CodebaseIndex
       @context_format = value
     end
 
+    # @param value [Boolean] Enable or disable the cache layer
+    # @raise [ConfigurationError] if value is not a boolean
+    def cache_enabled=(value)
+      validate_boolean!(:cache_enabled, value)
+      @cache_enabled = value
+    end
+
     # Add a gem to be indexed
     #
     # @param gem_name [String] Name of the gem
@@ -145,6 +154,14 @@ module CodebaseIndex
     def add_gem(gem_name, paths:, priority: :medium)
       @gem_configs[gem_name] = { paths: paths, priority: priority }
     end
+
+    private
+
+    def validate_boolean!(name, value)
+      return if value.is_a?(TrueClass) || value.is_a?(FalseClass)
+
+      raise ConfigurationError, "#{name} must be true or false, got #{value.inspect}"
+    end
   end
 
   # ════════════════════════════════════════════════════════════════════════
@@ -224,4 +241,6 @@ end
 
 require_relative 'codebase_index/builder'
 require_relative 'codebase_index/cost_model'
+require_relative 'codebase_index/cache/cache_store'
+require_relative 'codebase_index/cache/cache_middleware'
 require_relative 'codebase_index/railtie' if defined?(Rails::Railtie)

data/lib/tasks/codebase_index.rake

@@ -304,7 +304,7 @@ namespace :codebase_index do
     end
   end
 
-  desc 'Retrieve context for a query (for testing)'
+  # Internal debugging tool — hidden from `rails -T`
   task :retrieve, [:query] => :environment do |_t, args|
     query = args[:query] || raise('Usage: rake codebase_index:retrieve[query]')
 
@@ -400,7 +400,7 @@ namespace :codebase_index do
     puts "  Errors:    #{stats[:errors]}"
   end
 
-  desc "Analyze the gem's own source code and generate self-analysis output"
+  # Internal debugging tool — hidden from `rails -T`
   task :self_analyze do
     require 'digest'
     require 'json'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: codebase_index
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.1
 platform: ruby
 authors:
 - Leah Armstrong
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-02-28 00:00:00.000000000 Z
+date: 2026-03-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mcp
@@ -72,10 +72,15 @@ files:
 - lib/codebase_index/ast/node.rb
 - lib/codebase_index/ast/parser.rb
 - lib/codebase_index/builder.rb
+- lib/codebase_index/cache/cache_middleware.rb
+- lib/codebase_index/cache/cache_store.rb
+- lib/codebase_index/cache/redis_cache_store.rb
+- lib/codebase_index/cache/solid_cache_store.rb
 - lib/codebase_index/chunking/chunk.rb
 - lib/codebase_index/chunking/semantic_chunker.rb
 - lib/codebase_index/console/adapters/cache_adapter.rb
 - lib/codebase_index/console/adapters/good_job_adapter.rb
+- lib/codebase_index/console/adapters/job_adapter.rb
 - lib/codebase_index/console/adapters/sidekiq_adapter.rb
 - lib/codebase_index/console/adapters/solid_queue_adapter.rb
 - lib/codebase_index/console/audit_logger.rb
@@ -159,6 +164,7 @@ files:
 - lib/codebase_index/extractors/view_template_extractor.rb
 - lib/codebase_index/feedback/gap_detector.rb
 - lib/codebase_index/feedback/store.rb
+- lib/codebase_index/filename_utils.rb
 - lib/codebase_index/flow_analysis/operation_extractor.rb
 - lib/codebase_index/flow_analysis/response_code_mapper.rb
 - lib/codebase_index/flow_assembler.rb
@@ -170,6 +176,7 @@ files:
 - lib/codebase_index/formatting/gpt_adapter.rb
 - lib/codebase_index/formatting/human_adapter.rb
 - lib/codebase_index/graph_analyzer.rb
+- lib/codebase_index/mcp/bootstrapper.rb
 - lib/codebase_index/mcp/index_reader.rb
 - lib/codebase_index/mcp/renderers/claude_renderer.rb
 - lib/codebase_index/mcp/renderers/json_renderer.rb
@@ -184,6 +191,7 @@ files:
 - lib/codebase_index/notion/mappers/column_mapper.rb
 - lib/codebase_index/notion/mappers/migration_mapper.rb
 - lib/codebase_index/notion/mappers/model_mapper.rb
+- lib/codebase_index/notion/mappers/shared.rb
 - lib/codebase_index/notion/rate_limiter.rb
 - lib/codebase_index/observability/health_check.rb
 - lib/codebase_index/observability/instrumentation.rb
@@ -220,6 +228,7 @@ files:
 - lib/codebase_index/storage/qdrant.rb
 - lib/codebase_index/storage/vector_store.rb
 - lib/codebase_index/temporal/snapshot_store.rb
+- lib/codebase_index/token_utils.rb
 - lib/codebase_index/version.rb
 - lib/generators/codebase_index/install_generator.rb
 - lib/generators/codebase_index/pgvector_generator.rb