codebase_index 0.3.2 → 0.4.0

data/lib/codebase_index/storage/metadata_store.rb

@@ -1,196 +0,0 @@
-# frozen_string_literal: true
-
-require 'json'
-
-module CodebaseIndex
-  module Storage
-    # MetadataStore provides an interface for storing and querying unit metadata.
-    #
-    # All metadata store adapters must include the {Interface} module and implement
-    # its methods. The {SQLite} adapter is provided for local persistence.
-    #
-    # @example Using the SQLite adapter
-    #   store = CodebaseIndex::Storage::MetadataStore::SQLite.new(":memory:")
-    #   store.store("User", { type: "model", file_path: "app/models/user.rb" })
-    #   store.find("User")
-    #
-    module MetadataStore
-      # Interface that all metadata store adapters must implement.
-      module Interface
-        # Store or update metadata for a unit.
-        #
-        # @param id [String] Unique identifier for the unit
-        # @param metadata [Hash] Metadata to store
-        # @raise [NotImplementedError] if not implemented by adapter
-        def store(id, metadata)
-          raise NotImplementedError
-        end
-
-        # Find a unit by ID.
-        #
-        # @param id [String] The identifier to look up
-        # @return [Hash, nil] The stored metadata, or nil if not found
-        # @raise [NotImplementedError] if not implemented by adapter
-        def find(id)
-          raise NotImplementedError
-        end
-
-        # Find multiple units by IDs in a single query.
-        #
-        # Default implementation falls back to individual find calls.
-        # Adapters should override for batch-optimized behavior.
-        #
-        # @param ids [Array<String>] The identifiers to look up
-        # @return [Hash<String, Hash>] Map of id => metadata for found units
-        def find_batch(ids)
-          ids.each_with_object({}) do |id, result|
-            data = find(id)
-            result[id] = data if data
-          end
-        end
-
-        # Find all units of a given type.
-        #
-        # @param type [String] The unit type to filter by
-        # @return [Array<Hash>] Matching metadata records
-        # @raise [NotImplementedError] if not implemented by adapter
-        def find_by_type(type)
-          raise NotImplementedError
-        end
-
-        # Search metadata by text query across specified fields.
-        #
-        # @param query [String] Text to search for
-        # @param fields [Array<String>, nil] Specific fields to search (nil = all)
-        # @return [Array<Hash>] Matching metadata records
-        # @raise [NotImplementedError] if not implemented by adapter
-        def search(query, fields: nil)
-          raise NotImplementedError
-        end
-
-        # Delete a unit by ID.
-        #
-        # @param id [String] The identifier to delete
-        # @raise [NotImplementedError] if not implemented by adapter
-        def delete(id)
-          raise NotImplementedError
-        end
-
-        # Return the total number of stored units.
-        #
-        # @return [Integer] Total count
-        # @raise [NotImplementedError] if not implemented by adapter
-        def count
-          raise NotImplementedError
-        end
-      end
-
-      # SQLite-backed metadata store using the JSON1 extension.
-      #
-      # Stores unit metadata as JSON in a single table with type indexing
-      # for efficient filtering. Uses upsert semantics for store operations.
-      #
-      # @example
-      #   store = SQLite.new(":memory:")
-      #   store.store("User", { type: "model", namespace: "Admin" })
-      #   store.find("User")  # => { "type" => "model", "namespace" => "Admin" }
-      #
-      class SQLite
-        include Interface
-
-        # @param db_path [String] Path to the SQLite database file, or ":memory:" for in-memory
-        def initialize(db_path = ':memory:')
-          require 'sqlite3'
-          @db = ::SQLite3::Database.new(db_path)
-          @db.results_as_hash = true
-          create_table
-        end
-
-        # @see Interface#store
-        def store(id, metadata)
-          type = metadata[:type] || metadata['type']
-          data = JSON.generate(metadata)
-
-          @db.execute(<<~SQL, [id, type.to_s, data, Time.now.iso8601])
-            INSERT INTO units (id, type, data, updated_at) VALUES (?, ?, ?, ?)
-            ON CONFLICT(id) DO UPDATE SET
-              type = excluded.type, data = excluded.data, updated_at = excluded.updated_at
-          SQL
-        end
-
-        # @see Interface#find
-        def find(id)
-          row = @db.get_first_row('SELECT data FROM units WHERE id = ?', [id])
-          return nil unless row
-
-          JSON.parse(row['data'])
-        end
-
-        # @see Interface#find_batch
-        def find_batch(ids)
-          return {} if ids.empty?
-
-          placeholders = Array.new(ids.size, '?').join(', ')
-          rows = @db.execute("SELECT id, data FROM units WHERE id IN (#{placeholders})", ids)
-          rows.to_h do |row|
-            [row['id'], JSON.parse(row['data'])]
-          end
-        end
-
-        # @see Interface#find_by_type
-        def find_by_type(type)
-          rows = @db.execute('SELECT id, data FROM units WHERE type = ?', [type.to_s])
-          rows.map { |row| parse_row(row) }
-        end
-
-        # @see Interface#search
-        def search(query, fields: nil)
-          if fields
-            conditions = fields.map { "json_extract(data, '$.#{_1}') LIKE ?" }.join(' OR ')
-            params = fields.map { "%#{query}%" }
-            rows = @db.execute("SELECT id, data FROM units WHERE #{conditions}", params)
-          else
-            rows = @db.execute('SELECT id, data FROM units WHERE data LIKE ?', ["%#{query}%"])
-          end
-
-          rows.map { |row| parse_row(row) }
-        end
-
-        # @see Interface#delete
-        def delete(id)
-          @db.execute('DELETE FROM units WHERE id = ?', [id])
-        end
-
-        # @see Interface#count
-        def count
-          @db.get_first_value('SELECT COUNT(*) FROM units')
-        end
-
-        private
-
-        # Parse a database row into a metadata hash with the id field injected.
-        #
-        # @param row [Hash] Database row with 'id' and 'data' keys
-        # @return [Hash] Parsed metadata with 'id' key set
-        def parse_row(row)
-          parsed = JSON.parse(row['data'])
-          parsed['id'] = row['id']
-          parsed
-        end
-
-        # Create the units table if it doesn't exist.
-        def create_table
-          @db.execute(<<~SQL)
-            CREATE TABLE IF NOT EXISTS units (
-              id TEXT PRIMARY KEY,
-              type TEXT,
-              data JSON,
-              updated_at TEXT
-            )
-          SQL
-          @db.execute('CREATE INDEX IF NOT EXISTS idx_units_type ON units(type)')
-        end
-      end
-    end
-  end
-end

data/lib/codebase_index/storage/pgvector.rb

@@ -1,195 +0,0 @@
-# frozen_string_literal: true
-
-require 'json'
-require_relative 'vector_store'
-
-module CodebaseIndex
-  module Storage
-    module VectorStore
-      # PostgreSQL + pgvector adapter for vector storage and similarity search.
-      #
-      # Uses the pgvector extension for efficient approximate nearest neighbor
-      # search with HNSW indexing. Stores metadata as JSONB for flexible filtering.
-      #
-      # @example
-      #   store = Pgvector.new(connection: ActiveRecord::Base.connection, dimensions: 768)
-      #   store.ensure_schema!
-      #   store.store("User", [0.1, 0.2, ...], { type: "model" })
-      #   results = store.search([0.1, 0.2, ...], limit: 5, filters: { type: "model" })
-      #
-      class Pgvector # rubocop:disable Metrics/ClassLength
-        include Interface
-
-        TABLE = 'codebase_index_vectors'
-
-        # @param connection [Object] ActiveRecord database connection
-        # @param dimensions [Integer] Size of the embedding vectors
-        def initialize(connection:, dimensions:)
-          @connection = connection
-          @dimensions = dimensions
-        end
-
-        # Create the pgvector extension, vectors table, and HNSW index.
-        #
-        # Safe to call multiple times (uses IF NOT EXISTS).
-        def ensure_schema!
-          @connection.execute('CREATE EXTENSION IF NOT EXISTS vector')
-          @connection.execute(<<~SQL)
-            CREATE TABLE IF NOT EXISTS #{TABLE} (
-              id TEXT PRIMARY KEY,
-              embedding vector(#{@dimensions}),
-              metadata JSONB DEFAULT '{}',
-              created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
-            )
-          SQL
-          @connection.execute(<<~SQL)
-            CREATE INDEX IF NOT EXISTS idx_#{TABLE}_embedding_hnsw
-            ON #{TABLE} USING hnsw (embedding vector_cosine_ops)
-          SQL
-        end
-
-        # Store or update a vector with metadata.
-        #
-        # @param id [String] Unique identifier
-        # @param vector [Array<Float>] The embedding vector
-        # @param metadata [Hash] Optional metadata
-        # @see Interface#store
-        def store(id, vector, metadata = {})
-          validate_vector!(vector)
-          entry = format_entry(id, vector, metadata)
-
-          @connection.execute(<<~SQL)
-            INSERT INTO #{TABLE} (id, embedding, metadata, created_at)
-            VALUES #{entry}
-            ON CONFLICT (id) DO UPDATE SET
-              embedding = EXCLUDED.embedding,
-              metadata = EXCLUDED.metadata,
-              created_at = CURRENT_TIMESTAMP
-          SQL
-        end
-
-        # Store multiple vectors in a single multi-row INSERT.
-        #
-        # @param entries [Array<Hash>] Each entry has :id, :vector, :metadata keys
-        def store_batch(entries)
-          return if entries.empty?
-
-          values = entries.map do |entry|
-            validate_vector!(entry[:vector])
-            format_entry(entry[:id], entry[:vector], entry[:metadata] || {})
-          end
-
-          @connection.execute(<<~SQL)
-            INSERT INTO #{TABLE} (id, embedding, metadata, created_at)
-            VALUES #{values.join(",\n")}
-            ON CONFLICT (id) DO UPDATE SET
-              embedding = EXCLUDED.embedding,
-              metadata = EXCLUDED.metadata,
-              created_at = CURRENT_TIMESTAMP
-          SQL
-        end
-
-        # Search for similar vectors using cosine distance.
-        #
-        # @param query_vector [Array<Float>] The query embedding
-        # @param limit [Integer] Maximum results to return
-        # @param filters [Hash] Metadata key-value filters
-        # @return [Array<SearchResult>] Results sorted by descending similarity
-        # @see Interface#search
-        def search(query_vector, limit: 10, filters: {})
-          validate_vector!(query_vector)
-          vector_literal = "[#{query_vector.join(',')}]"
-          where_clause = build_where(filters)
-
-          sql = <<~SQL
-            SELECT id, embedding <=> '#{vector_literal}' AS distance, metadata
-            FROM #{TABLE}
-            #{where_clause}
-            ORDER BY distance ASC
-            LIMIT #{limit.to_i}
-          SQL
-
-          rows = @connection.execute(sql)
-          rows.map { |row| row_to_result(row) }
-        end
-
-        # @see Interface#delete
-        def delete(id)
-          quoted_id = @connection.quote(id)
-          @connection.execute("DELETE FROM #{TABLE} WHERE id = #{quoted_id}")
-        end
-
-        # @see Interface#delete_by_filter
-        def delete_by_filter(filters)
-          where_clause = build_where(filters)
-          @connection.execute("DELETE FROM #{TABLE} #{where_clause}")
-        end
-
-        # @see Interface#count
-        def count
-          result = @connection.execute("SELECT COUNT(*) AS count FROM #{TABLE}")
-          result.first['count'].to_i
-        end
-
-        private
-
-        # Format a single entry as a SQL VALUES tuple.
-        #
-        # @param id [String] Unique identifier
-        # @param vector [Array<Float>] Embedding vector
-        # @param metadata [Hash] Entry metadata
-        # @return [String] SQL values row literal
-        def format_entry(id, vector, metadata)
-          quoted_id = @connection.quote(id)
-          quoted_metadata = @connection.quote(JSON.generate(metadata))
-          vector_literal = "[#{vector.join(',')}]"
-          "(#{quoted_id}, '#{vector_literal}', #{quoted_metadata}::jsonb, CURRENT_TIMESTAMP)"
-        end
-
-        # Convert a database row to a SearchResult.
-        #
-        # @param row [Hash] Database row with id, distance, metadata
-        # @return [SearchResult]
-        def row_to_result(row)
-          metadata = row['metadata']
-          parsed_metadata = metadata.is_a?(String) ? JSON.parse(metadata) : metadata
-          SearchResult.new(
-            id: row['id'],
-            score: 1.0 - row['distance'].to_f,
-            metadata: parsed_metadata
-          )
-        end
-
-        # Build a WHERE clause from metadata filters.
-        #
-        # @param filters [Hash] Metadata key-value pairs
-        # @return [String] SQL WHERE clause, or empty string if no filters
-        def build_where(filters)
-          return '' if filters.empty?
-
-          conditions = filters.map do |key, value|
-            key_s = key.to_s
-            unless key_s.match?(/\A[a-zA-Z_][a-zA-Z0-9_]*\z/)
-              raise ArgumentError, "Invalid filter key: #{key_s.inspect}"
-            end
-
-            "metadata->>'#{key_s}' = #{@connection.quote(value.to_s)}"
-          end
-          "WHERE #{conditions.join(' AND ')}"
-        end
-
-        # Validate that all vector elements are numeric.
-        #
-        # @param vector [Array] The vector to validate
-        # @raise [ArgumentError] if any element is not numeric
-        def validate_vector!(vector)
-          vector.each_with_index do |element, i|
-            unless element.is_a?(Numeric)
-              raise ArgumentError, "Vector element at index #{i} is not numeric: #{element.inspect}"
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/codebase_index/storage/qdrant.rb

@@ -1,205 +0,0 @@
-# frozen_string_literal: true
-
-require 'net/http'
-require 'json'
-require 'uri'
-require_relative 'vector_store'
-
-module CodebaseIndex
-  module Storage
-    module VectorStore
-      # Qdrant adapter for vector storage and similarity search via HTTP API.
-      #
-      # Communicates with a Qdrant instance over HTTP. Supports optional API key
-      # authentication for managed/cloud deployments.
-      #
-      # @example
-      #   store = Qdrant.new(url: "http://localhost:6333", collection: "codebase")
-      #   store.ensure_collection!(dimensions: 768)
-      #   store.store("User", [0.1, 0.2, ...], { type: "model" })
-      #   results = store.search([0.1, 0.2, ...], limit: 5)
-      #
-      class Qdrant # rubocop:disable Metrics/ClassLength
-        include Interface
-
-        # @param url [String] Qdrant server URL
-        # @param collection [String] Collection name
-        # @param api_key [String, nil] Optional API key for authentication
-        def initialize(url:, collection:, api_key: nil)
-          @url = url
-          @collection = collection
-          @api_key = api_key
-          @uri = URI(url)
-        end
-
-        # Create the collection if it doesn't exist.
-        #
-        # @param dimensions [Integer] Vector dimensionality
-        def ensure_collection!(dimensions:)
-          body = {
-            vectors: {
-              size: dimensions,
-              distance: 'Cosine'
-            }
-          }
-          request(:put, "/collections/#{@collection}", body)
-        end
-
-        # Store or update a vector with metadata payload.
-        #
-        # @param id [String] Unique identifier
-        # @param vector [Array<Float>] The embedding vector
-        # @param metadata [Hash] Optional payload metadata
-        # @see Interface#store
-        def store(id, vector, metadata = {})
-          body = {
-            points: [
-              {
-                id: id,
-                vector: vector,
-                payload: metadata
-              }
-            ]
-          }
-          request(:put, "/collections/#{@collection}/points", body)
-        end
-
-        # Store multiple vectors in a single batch upsert request.
-        #
-        # Sends the entire entries array in one HTTP call. Callers are responsible
-        # for chunking into reasonable batch sizes (e.g., 100–500 points) before
-        # calling this method; the embedding Indexer's +batch_size+ config controls
-        # the upstream chunk size.
-        #
-        # @param entries [Array<Hash>] Each entry has :id, :vector, :metadata keys
-        def store_batch(entries)
-          return if entries.empty?
-
-          body = {
-            points: entries.map do |entry|
-              { id: entry[:id], vector: entry[:vector], payload: entry[:metadata] || {} }
-            end
-          }
-          request(:put, "/collections/#{@collection}/points", body)
-        end
-
-        # Search for similar vectors.
-        #
-        # @param query_vector [Array<Float>] The query embedding
-        # @param limit [Integer] Maximum results to return
-        # @param filters [Hash] Metadata key-value filters
-        # @return [Array<SearchResult>] Results sorted by descending similarity
-        # @see Interface#search
-        def search(query_vector, limit: 10, filters: {})
-          body = {
-            vector: query_vector,
-            limit: limit,
-            with_payload: true
-          }
-          body[:filter] = build_filter(filters) unless filters.empty?
-
-          response = request(:post, "/collections/#{@collection}/points/search", body)
-          results = response['result'] || []
-
-          results.map do |hit|
-            SearchResult.new(
-              id: hit['id'],
-              score: hit['score'],
-              metadata: hit['payload']
-            )
-          end
-        end
-
-        # @see Interface#delete
-        def delete(id)
-          body = { points: [id] }
-          request(:post, "/collections/#{@collection}/points/delete", body)
-        end
-
-        # @see Interface#delete_by_filter
-        def delete_by_filter(filters)
-          body = { filter: build_filter(filters) }
-          request(:post, "/collections/#{@collection}/points/delete", body)
-        end
-
-        # @see Interface#count
-        def count
-          response = request(:post, "/collections/#{@collection}/points/count", { exact: true })
-          response['result']['count']
-        end
-
-        private
-
-        # Build a Qdrant filter from metadata key-value pairs.
-        #
-        # @param filters [Hash] Metadata filters
-        # @return [Hash] Qdrant-compatible filter with must conditions
-        def build_filter(filters)
-          conditions = filters.map do |key, value|
-            { key: key.to_s, match: { value: value } }
-          end
-          { must: conditions }
-        end
-
-        # Send an HTTP request to the Qdrant API.
-        #
-        # @param method [Symbol] HTTP method (:get, :post, :put, :delete)
-        # @param path [String] API path
-        # @param body [Hash, nil] Request body
-        # @return [Hash] Parsed JSON response
-        # @raise [CodebaseIndex::Error] if the API returns a non-success status
-        def request(method, path, body = nil)
-          req = build_request(method, path, body)
-          response = http_client.request(req)
-
-          unless response.is_a?(Net::HTTPSuccess)
-            raise CodebaseIndex::Error, "Qdrant API error: #{response.code} #{response.body}"
-          end
-
-          JSON.parse(response.body)
-        rescue Errno::ECONNRESET, Net::OpenTimeout, IOError
-          # Connection dropped — reset and retry once
-          @http_client = nil
-          response = http_client.request(req)
-          unless response.is_a?(Net::HTTPSuccess)
-            raise CodebaseIndex::Error, "Qdrant API error: #{response.code} #{response.body}"
-          end
-
-          JSON.parse(response.body)
-        end
-
-        # Return a reusable, started HTTP client for the Qdrant server.
-        # Calling http.start opens a persistent TCP connection so
-        # keep_alive_timeout actually takes effect across requests.
-        #
-        # @return [Net::HTTP]
-        def http_client
-          return @http_client if @http_client&.started?
-
-          http = Net::HTTP.new(@uri.host, @uri.port)
-          http.use_ssl = @uri.scheme == 'https'
-          http.open_timeout = 10
-          http.read_timeout = 30
-          http.keep_alive_timeout = 30
-          http.start
-          @http_client = http
-        end
-
-        # Build an HTTP request with headers and body.
-        #
-        # @param method [Symbol] HTTP method
-        # @param path [String] API path
-        # @param body [Hash, nil] Request body
-        # @return [Net::HTTPRequest]
-        def build_request(method, path, body)
-          request_class = { get: Net::HTTP::Get, post: Net::HTTP::Post,
-                            put: Net::HTTP::Put, delete: Net::HTTP::Delete }.fetch(method)
-          req = request_class.new(path, 'Content-Type' => 'application/json')
-          req['api-key'] = @api_key if @api_key
-          req.body = body.to_json if body
-          req
-        end
-      end
-    end
-  end
-end