boxcars 0.2.10 → 0.2.12

data/lib/boxcars/boxcar/vector_stores/hnswlib/hnswlib_search.rb

@@ -1,54 +0,0 @@
-# frozen_string_literal: true
-
-require 'hnswlib'
-require 'json'
-
-module Boxcars
-  module VectorStores
-    module Hnswlib
-      class HnswlibSearch
-        def initialize(vector_store:, options: {})
-          validate_params(vector_store)
-          @vector_store = vector_store
-          @json_doc_path = options[:json_doc_path]
-          @num_neighbors = options[:num_neighbors] || 1
-        end
-
-        def call(query)
-          search(query)
-        end
-
-        private
-
-        attr_reader :json_doc_path, :vector_store, :num_neighbors
-
-        def validate_params(vector_store)
-          raise_error 'vector_store must be an Hnswlib::HierarchicalNSW' unless vector_store.is_a?(::Hnswlib::HierarchicalNSW)
-        end
-
-        def search(query)
-          raw_results = vector_store.search_knn(query, num_neighbors)
-          raw_results.map { |doc_id, distance| lookup_embedding2(doc_id, distance) }.compact
-        end
-
-        def lookup_embedding2(doc_id, distance)
-          embedding_data = parsed_data.find { |embedding| embedding[:doc_id] == doc_id }
-          return unless embedding_data
-
-          { document: embedding_data[:document], distance: distance }
-        end
-
-        def parsed_data
-          @parsed_data ||= JSON.parse(
-            File.read(json_doc_path),
-            symbolize_names: true
-          )
-        end
-
-        def raise_error(message)
-          raise ::Boxcars::ArgumentError, message
-        end
-      end
-    end
-  end
-end

data/lib/boxcars/boxcar/vector_stores/hnswlib/save_to_hnswlib.rb

@@ -1,80 +0,0 @@
-# frozen_string_literal: true
-
-require 'hnswlib'
-require 'json'
-require 'fileutils'
-
-module Boxcars
-  module VectorStores
-    module Hnswlib
-      class SaveToHnswlib
-        include VectorStore
-
-        # @param document_embeddings [Array] An array of hashes containing the document id, document text, and embedding.
-        # @param index_file_path [String] The path to the index file.
-        # @param hnswlib_config [Boxcars::VectorStores::Hnswlib::Config] The config object for the hnswlib index.
-        # @option json_doc_file_path [String] Optional. The path to the json file containing the document text.
-        def initialize(document_embeddings:, index_file_path:, hnswlib_config:, json_doc_file_path: nil)
-          @document_embeddings = document_embeddings
-          @index_file_path = index_file_path
-          @json_doc_file_path = json_doc_file_path || index_file_path.gsub(/\.bin$/, '.json')
-
-          @hnswlib_config = hnswlib_config
-          @index = ::Hnswlib::HnswIndex.new(
-            n_features: hnswlib_config.dim,
-            max_item: hnswlib_config.max_item,
-            metric: hnswlib_config.metric
-          )
-        end
-
-        def call
-          validate_params
-          document_texts = []
-
-          document_embeddings.each do |embedding|
-            index.add_item(embedding[:doc_id], embedding[:embedding])
-
-            document_texts << { doc_id: embedding[:doc_id], embedding: embedding[:embedding], document: embedding[:document] }
-          end
-
-          write_files(index, document_texts)
-        end
-
-        private
-
-        def write_files(index, document_texts)
-          FileUtils.mkdir_p(File.dirname(json_doc_file_path))
-          File.write(json_doc_file_path, document_texts.to_json)
-
-          FileUtils.mkdir_p(File.dirname(index_file_path))
-          File.write("#{File.dirname(index_file_path)}/hnswlib_config.json", hnswlib_config.to_json)
-
-          index.save(index_file_path)
-        end
-
-        attr_reader :index, :document_embeddings, :index_file_path, :json_doc_file_path, :hnswlib_config
-
-        def validate_params
-          raise_error("document_embeddings must be an array") unless document_embeddings.is_a?(Array)
-          raise_error("dim must be an integer") unless hnswlib_config.dim.is_a?(Integer)
-          raise_error("index_file_path must be a string") unless index_file_path.is_a?(String)
-
-          [index_file_path, json_doc_file_path].each do |path|
-            check_parent_directory(path)
-          end
-        end
-
-        def check_parent_directory(path)
-          return unless path
-
-          parent_dir = File.dirname(path)
-          raise_error('parent directory must exist') unless File.directory?(parent_dir)
-        end
-
-        def raise_error(message)
-          raise ::Boxcars::ValueError, message
-        end
-      end
-    end
-  end
-end

data/lib/boxcars/boxcar/vector_stores/in_memory/add_documents.rb

@@ -1,67 +0,0 @@
-# frozen_string_literal: true
-
-module Boxcars
-  module VectorStores
-    module InMemory
-      MemoryVector = Struct.new(:content, :embedding, :metadatax)
-
-      class AddDocuments
-        include VectorStore
-
-        def initialize(embedding_tool: :openai, documents: nil)
-          validate_params(embedding_tool, documents)
-          @embedding_tool = embedding_tool
-          @documents = documents
-          @memory_vectors = []
-        end
-
-        def call
-          texts = @documents.map { |doc| doc[:page_content] }
-          vectors = generate_vectors(texts)
-          add_vectors(vectors, @documents)
-          @memory_vectors
-        end
-
-        private
-
-        def validate_params(embedding_tool, documents)
-          raise ::Boxcars::ArgumentError, 'documents is nil' unless documents
-          return if %i[openai tensorflow].include?(embedding_tool)
-
-          raise ::Boxcars::ArgumentError, 'embedding_tool is invalid'
-        end
-
-        # returns array of documents with vectors
-        def add_vectors(vectors, documents)
-          vectors.zip(documents).each do |vector, doc|
-            memory_vector = MemoryVector.new(doc[:page_content], vector, doc[:metadata])
-            @memory_vectors << memory_vector
-          end
-        end
-
-        def generate_vectors(texts)
-          embeddings_method[:klass].call(
-            texts: texts, client: embeddings_method[:client]
-          )
-        end
-
-        def embeddings_method
-          @embeddings_method ||=
-            case @embedding_tool
-            when :openai
-              { klass: Boxcars::VectorStores::EmbedViaOpenAI, client: openai_client }
-            when :tensorflow
-              { klass: Boxcars::VectorStores::EmbedViaTensorflow, client: nil }
-            end
-        end
-
-        # Get the OpenAI client
-        # @param openai_access_token [String] the OpenAI access token
-        # @return [OpenAI::Client]
-        def openai_client(openai_access_token: nil)
-          @openai_client ||= Openai.open_ai_client(openai_access_token: openai_access_token)
-        end
-      end
-    end
-  end
-end

data/lib/boxcars/boxcar/vector_stores/in_memory/search.rb

@@ -1,81 +0,0 @@
-# frozen_string_literal: true
-
-# require 'openai'
-#
-# documents = [
-#     { page_content: "hello", metadata: { a: 1 } },
-#     { page_content: "hi", metadata: { a: 1 } },
-#     { page_content: "bye", metadata: { a: 1 } },
-#     { page_content: "what's this", metadata: { a: 1 } },
-# ]
-#
-# vector_documents = Boxcars::VectorStores::InMemory::AddDocuments.call(embedding_tool: :openai, documents: documents)
-#
-# result = Boxcars::VectorStores::InMemory::Search.call(vecotr_documents: vector_documents, query: "hello")
-#
-# expect(result).to eq(Boxcars::VectorStores::Document.new({ page_content: "hello", metadata: { a: 1 } }))
-
-module Boxcars
-  module VectorStores
-    module InMemory
-      class Search
-        include VectorStore
-        def initialize(vector_documents:, query:, embedding_tool: :openai)
-          validate_params(vector_documents, query, embedding_tool)
-          @vector_documents = vector_documents
-          @query = query
-          @embedding_tool = embedding_tool
-        end
-
-        def call
-          results = @vector_documents.map do |doc|
-            {
-              document: doc,
-              similarity: cosine_similarity(query_vector, doc[:vector])
-            }
-          end
-          results.min_by { |result| -result[:similarity] }[:document]
-        end
-
-        private
-
-        def validate_params(vector_documents, query, embedding_tool)
-          raise ::Boxcars::ArgumentError, 'query is empty' if query.to_s.empty?
-          raise ::Boxcars::ArgumentError, 'embedding_tool is invalid' unless %i[openai tensorflow].include?(embedding_tool)
-
-          unless vector_documents.is_a?(Array) && vector_documents.all? do |doc|
-                   doc.is_a?(Hash) && doc.key?(:document) && doc.key?(:vector)
-                 end
-            raise ::Boxcars::ArgumentError, "vector_documents is not valid"
-          end
-        end
-
-        def query_vector
-          embeddings_method(@embedding_tool)[:klass].call(
-            texts: [@query], client: embeddings_method(@embedding_tool)[:client]
-          ).first
-        end
-
-        def openai_client(openai_access_token: nil)
-          @openai_client ||= Openai.open_ai_client(openai_access_token: openai_access_token)
-        end
-
-        def embeddings_method(embedding_tool)
-          case embedding_tool
-          when :openai
-            { klass: Boxcars::VectorStores::EmbedViaOpenAI, client: openai_client }
-          when :tensorflow
-            { klass: Boxcars::VectorStores::EmbedViaTensorflow, client: nil }
-          end
-        end
-
-        def cosine_similarity(vector1, vector2)
-          dot_product = vector1.zip(vector2).reduce(0) { |sum, (a, b)| sum + (a * b) }
-          magnitude1 = Math.sqrt(vector1.reduce(0) { |sum, a| sum + (a**2) })
-          magnitude2 = Math.sqrt(vector2.reduce(0) { |sum, b| sum + (b**2) })
-          dot_product / (magnitude1 * magnitude2)
-        end
-      end
-    end
-  end
-end

data/lib/boxcars/boxcar/vector_stores/similarity_search.rb

@@ -1,55 +0,0 @@
-# frozen_string_literal: true
-
-require 'hnswlib'
-
-module Boxcars
-  module VectorStores
-    class SimilaritySearch
-      def initialize(embeddings:, vector_store:, openai_connection: nil, openai_access_token: nil)
-        @embeddings = embeddings
-        @vector_store = vector_store
-        @similarity_search_instance = create_similarity_search_instance
-        @openai_connection = openai_connection || default_connection(openai_access_token: openai_access_token)
-      end
-
-      def call(query:)
-        validate_query(query)
-        query_vector = convert_query_to_vector(query)
-        @similarity_search_instance.call(query_vector)
-      end
-
-      private
-
-      attr_reader :embeddings, :vector_store, :openai_connection
-
-      def default_connection(openai_access_token: nil)
-        Openai.open_ai_client(openai_access_token: openai_access_token)
-      end
-
-      def validate_query(query)
-        raise_error 'query must be a string' unless query.is_a?(String)
-        raise_error 'query must not be empty' if query.empty?
-      end
-
-      def convert_query_to_vector(query)
-        Boxcars::VectorStores::EmbedViaOpenAI.call(texts: [query], client: openai_connection).first[:embedding]
-      end
-
-      def create_similarity_search_instance
-        case vector_store
-        when ::Hnswlib::HierarchicalNSW
-          Boxcars::VectorStores::Hnswlib::HnswlibSearch.new(
-            vector_store: vector_store,
-            options: { json_doc_path: embeddings, num_neighbors: 2 }
-          )
-        else
-          raise_error 'Unsupported vector store provided'
-        end
-      end
-
-      def raise_error(message)
-        raise ArgumentError, message
-      end
-    end
-  end
-end