boxcars 0.2.11 → 0.2.12

data/lib/boxcars/vector_store/hnswlib/hnswlib_config.rb

@@ -1,56 +0,0 @@
-# frozen_string_literal: true
-
-require 'json'
-
-module Boxcars
-  module VectorStore
-    module Hnswlib
-      class HnswlibConfig
-        attr_reader :metric, :max_item, :dim, :ef_construction, :m
-
-        # used for search index.
-        #
-        # @param max_item [Integer] The maximum number of items.
-        #
-        # @param metric [String] The distance metric between vectors ('l2', 'dot', or 'cosine').
-        #
-        # @param ef_construction [Integer] The size of the dynamic list for the nearest neighbors.
-        #                        It controls the index time/accuracy trade-off.
-        #
-        # @param max_outgoing_connection [Integer] The maximum number of outgoing connections in the graph
-        #
-        # reference: https://yoshoku.github.io/hnswlib.rb/doc/
-        def initialize(
-          metric: "l2",
-          max_item: 10000,
-          dim: 2,
-          ef_construction: 200,
-          max_outgoing_connection: 16
-        )
-          @metric = metric
-          @max_item = max_item
-          @dim = dim
-          @ef_construction = ef_construction
-          @max_outgoing_connection = max_outgoing_connection
-        end
-
-        def space
-          @metric == 'dot' ? 'ip' : 'l2'
-        end
-
-        def to_json(*args)
-          JSON.pretty_generate(
-            {
-              metric: @metric,
-              max_item: @max_item,
-              dim: @dim,
-              ef_construction: @ef_construction,
-              max_outgoing_connection: @max_outgoing_connection
-            },
-            *args
-          )
-        end
-      end
-    end
-  end
-end

data/lib/boxcars/vector_store/hnswlib/hnswlib_search.rb

@@ -1,54 +0,0 @@
-# frozen_string_literal: true
-
-require 'hnswlib'
-require 'json'
-
-module Boxcars
-  module VectorStore
-    module Hnswlib
-      class HnswlibSearch
-        def initialize(vector_store:, options: {})
-          validate_params(vector_store)
-          @vector_store = vector_store
-          @json_doc_path = options[:json_doc_path]
-          @num_neighbors = options[:num_neighbors] || 1
-        end
-
-        def call(query)
-          search(query)
-        end
-
-        private
-
-        attr_reader :json_doc_path, :vector_store, :num_neighbors
-
-        def validate_params(vector_store)
-          raise_error 'vector_store must be an Hnswlib::HierarchicalNSW' unless vector_store.is_a?(::Hnswlib::HierarchicalNSW)
-        end
-
-        def search(query)
-          raw_results = vector_store.search_knn(query, num_neighbors)
-          raw_results.map { |doc_id, distance| lookup_embedding2(doc_id, distance) }.compact
-        end
-
-        def lookup_embedding2(doc_id, distance)
-          embedding_data = parsed_data.find { |embedding| embedding[:doc_id] == doc_id }
-          return unless embedding_data
-
-          { document: embedding_data[:document], distance: distance }
-        end
-
-        def parsed_data
-          @parsed_data ||= JSON.parse(
-            File.read(json_doc_path),
-            symbolize_names: true
-          )
-        end
-
-        def raise_error(message)
-          raise ::Boxcars::ArgumentError, message
-        end
-      end
-    end
-  end
-end

data/lib/boxcars/vector_store/in_memory/add_documents.rb

@@ -1,67 +0,0 @@
-# frozen_string_literal: true
-
-module Boxcars
-  module VectorStore
-    module InMemory
-      MemoryVector = Struct.new(:content, :embedding, :metadatax)
-
-      class AddDocuments
-        include VectorStore
-
-        def initialize(embedding_tool: :openai, documents: nil)
-          validate_params(embedding_tool, documents)
-          @embedding_tool = embedding_tool
-          @documents = documents
-          @memory_vectors = []
-        end
-
-        def call
-          texts = @documents.map { |doc| doc[:page_content] }
-          vectors = generate_vectors(texts)
-          add_vectors(vectors, @documents)
-          @memory_vectors
-        end
-
-        private
-
-        def validate_params(embedding_tool, documents)
-          raise ::Boxcars::ArgumentError, 'documents is nil' unless documents
-          return if %i[openai tensorflow].include?(embedding_tool)
-
-          raise ::Boxcars::ArgumentError, 'embedding_tool is invalid'
-        end
-
-        # returns array of documents with vectors
-        def add_vectors(vectors, documents)
-          vectors.zip(documents).each do |vector, doc|
-            memory_vector = MemoryVector.new(doc[:page_content], vector, doc[:metadata])
-            @memory_vectors << memory_vector
-          end
-        end
-
-        def generate_vectors(texts)
-          embeddings_method[:klass].call(
-            texts: texts, client: embeddings_method[:client]
-          )
-        end
-
-        def embeddings_method
-          @embeddings_method ||=
-            case @embedding_tool
-            when :openai
-              { klass: Boxcars::VectorStore::EmbedViaOpenAI, client: openai_client }
-            when :tensorflow
-              { klass: Boxcars::VectorStore::EmbedViaTensorflow, client: nil }
-            end
-        end
-
-        # Get the OpenAI client
-        # @param openai_access_token [String] the OpenAI access token
-        # @return [OpenAI::Client]
-        def openai_client(openai_access_token: nil)
-          @openai_client ||= Openai.open_ai_client(openai_access_token: openai_access_token)
-        end
-      end
-    end
-  end
-end

data/lib/boxcars/vector_store/similarity_search.rb

@@ -1,55 +0,0 @@
-# frozen_string_literal: true
-
-require 'hnswlib'
-
-module Boxcars
-  module VectorStore
-    class SimilaritySearch
-      def initialize(embeddings:, vector_store:, openai_connection: nil, openai_access_token: nil)
-        @embeddings = embeddings
-        @vector_store = vector_store
-        @similarity_search_instance = create_similarity_search_instance
-        @openai_connection = openai_connection || default_connection(openai_access_token: openai_access_token)
-      end
-
-      def call(query:)
-        validate_query(query)
-        query_vector = convert_query_to_vector(query)
-        @similarity_search_instance.call(query_vector)
-      end
-
-      private
-
-      attr_reader :embeddings, :vector_store, :openai_connection
-
-      def default_connection(openai_access_token: nil)
-        Openai.open_ai_client(openai_access_token: openai_access_token)
-      end
-
-      def validate_query(query)
-        raise_error 'query must be a string' unless query.is_a?(String)
-        raise_error 'query must not be empty' if query.empty?
-      end
-
-      def convert_query_to_vector(query)
-        Boxcars::VectorStore::EmbedViaOpenAI.call(texts: [query], client: openai_connection).first[:embedding]
-      end
-
-      def create_similarity_search_instance
-        case vector_store
-        when ::Hnswlib::HierarchicalNSW
-          Boxcars::VectorStore::Hnswlib::HnswlibSearch.new(
-            vector_store: vector_store,
-            options: { json_doc_path: embeddings, num_neighbors: 2 }
-          )
-        else
-          raise_error 'Unsupported vector store provided'
-        end
-      end
-
-      def raise_error(message)
-        raise ArgumentError, message
-      end
-    end
-  end
-end