boxcars 0.2.11 → 0.2.13

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: boxcars
 version: !ruby/object:Gem::Version
-  version: 0.2.11
+  version: 0.2.13
 platform: ruby
 authors:
 - Francis Sullivan
@@ -9,92 +9,78 @@ authors:
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-05-05 00:00:00.000000000 Z
+date: 2023-05-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: debug
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.1'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.1'
-- !ruby/object:Gem::Dependency
-  name: dotenv
+  name: google_search_results
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.8'
-  type: :development
+        version: '2.2'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.8'
+        version: '2.2'
 - !ruby/object:Gem::Dependency
-  name: rspec
+  name: gpt4all
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.2'
-  type: :development
+        version: 0.0.4
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.2'
+        version: 0.0.4
 - !ruby/object:Gem::Dependency
-  name: google_search_results
+  name: hnswlib
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.2'
+        version: '0.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.2'
+        version: '0.8'
 - !ruby/object:Gem::Dependency
-  name: gpt4all
+  name: ruby-openai
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.0.4
+        version: '4.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.0.4
+        version: '4.1'
 - !ruby/object:Gem::Dependency
-  name: ruby-openai
+  name: pgvector
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '0.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '0.2'
 description: You simply set an OpenAI key, give a number of Boxcars to a Train, and
   magic ensues when you run it.
 email:
@@ -124,6 +110,7 @@ files:
 - lib/boxcars/boxcar/google_search.rb
 - lib/boxcars/boxcar/sql.rb
 - lib/boxcars/boxcar/swagger.rb
+- lib/boxcars/boxcar/vector_answer.rb
 - lib/boxcars/boxcar/wikipedia_search.rb
 - lib/boxcars/conversation.rb
 - lib/boxcars/conversation_prompt.rb
@@ -144,13 +131,17 @@ files:
 - lib/boxcars/vector_store/document.rb
 - lib/boxcars/vector_store/embed_via_open_ai.rb
 - lib/boxcars/vector_store/embed_via_tensorflow.rb
-- lib/boxcars/vector_store/hnswlib/build_vector_store.rb
-- lib/boxcars/vector_store/hnswlib/hnswlib_config.rb
-- lib/boxcars/vector_store/hnswlib/hnswlib_search.rb
+- lib/boxcars/vector_store/hnswlib/build_from_files.rb
+- lib/boxcars/vector_store/hnswlib/load_from_disk.rb
 - lib/boxcars/vector_store/hnswlib/save_to_hnswlib.rb
-- lib/boxcars/vector_store/in_memory/add_documents.rb
+- lib/boxcars/vector_store/hnswlib/search.rb
+- lib/boxcars/vector_store/in_memory/build_from_document_array.rb
+- lib/boxcars/vector_store/in_memory/build_from_files.rb
 - lib/boxcars/vector_store/in_memory/search.rb
-- lib/boxcars/vector_store/similarity_search.rb
+- lib/boxcars/vector_store/pgvector/build_from_array.rb
+- lib/boxcars/vector_store/pgvector/build_from_files.rb
+- lib/boxcars/vector_store/pgvector/save_to_database.rb
+- lib/boxcars/vector_store/pgvector/search.rb
 - lib/boxcars/vector_store/split_text.rb
 - lib/boxcars/version.rb
 homepage: https://github.com/BoxcarsAI/boxcars
@@ -169,14 +160,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.6.0
+      version: '3.0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.10
+rubygems_version: 3.2.32
 signing_key:
 specification_version: 4
 summary: Boxcars is a gem that enables you to create new systems with AI composability.

data/lib/boxcars/vector_store/hnswlib/build_vector_store.rb

@@ -1,157 +0,0 @@
-# frozen_string_literal: true
-
-require 'fileutils'
-require 'hnswlib'
-require 'json'
-
-module Boxcars
-  module VectorStore
-    module Hnswlib
-      class BuildVectorStore
-        include VectorStore
-
-        # This class is responsible for building the vector store for the hnswlib similarity search.
-        # It will load the training data, generate the embeddings, and save the vector store.
-        # It will also load the vector store into memory.
-        # For later use, it will save the splitted document with index numbers to a json file.
-        #
-        # @param training_data_path [String] The path to the training data. Can be a glob pattern.
-        # @param index_file_path [String] The path to the index file.
-        # @param split_chunk_size [Integer] The number of documents to split the text into. default 2000
-        # @option json_doc_file_path [String]. The json file containing the document text.
-        #                                      if nil, it will reuse index file name.
-        # @option force_rebuild [Boolean] Optional. If true, will rebuild the index even if it already exists.
-        def initialize(
-          training_data_path:,
-          index_file_path:,
-          split_chunk_size: 2000,
-          json_doc_file_path: nil,
-          force_rebuild: true
-        )
-          @training_data_path = training_data_path
-          @index_file_path = index_file_path
-          @split_chunk_size = split_chunk_size
-          @json_doc_file_path = json_doc_file_path || index_file_path.gsub(/\.bin$/, '.json')
-          @force_rebuild = force_rebuild
-        end
-
-        def call
-          validate_params
-          data = load_files
-          documents = split_text_into_chunks(data)
-          embeddings_with_config = generate_embeddings(documents)
-          save_vector_store(embeddings_with_config)
-          load_hnsw
-        end
-
-        private
-
-        attr_reader :training_data_path, :index_file_path, :split_chunk_size, :json_doc_file_path, :force_rebuild
-
-        def validate_params
-          training_data_dir = File.dirname(training_data_path.gsub(/\*{1,2}/, ''))
-          raise_error('training_data_path parent directory must exist') unless File.directory?(training_data_dir)
-          raise_error('No files found at the training_data_path pattern') if Dir.glob(training_data_path).empty?
-
-          index_dir = File.dirname(index_file_path)
-          raise_error('index_file_path parent directory must exist') unless File.directory?(index_dir)
-
-          raise_error('split_chunk_size must be an integer') unless split_chunk_size.is_a?(Integer)
-        end
-
-        def load_files
-          data = []
-          files = Dir.glob(training_data_path)
-          raise_error "No files found at #{training_data_path}" if files.empty?
-
-          files.each do |file|
-            data << File.read(file)
-          end
-          puts "Added #{files.length} files to data. Splitting text into chunks..."
-          data
-        end
-
-        def split_text_into_chunks(data)
-          return true unless rebuild_required?
-
-          docs = []
-          data.each do |chunk|
-            doc_output = Boxcars::VectorStore::SplitText.call(
-              separator: "\n", chunk_size: split_chunk_size, chunk_overlap: 0, text: chunk
-            )
-            docs.concat(doc_output)
-          end
-          docs
-        end
-
-        def rebuild_required?
-          hnswlib_config_json = "#{File.dirname(index_file_path)}/hnswlib_config.json"
-          return true unless File.exist?(index_file_path)
-          return true if File.exist?(index_file_path) && !File.exist?(hnswlib_config_json)
-          return true if force_rebuild
-
-          false
-        end
-
-        def generate_embeddings(documents)
-          return true unless rebuild_required?
-
-          puts "Initializing Store..."
-          openai_client = Openai.open_ai_client
-          embeddings_with_dim = Boxcars::VectorStore::EmbedViaOpenAI.call(texts: documents, client: openai_client)
-          document_embeddings = embeddings_with_dim.map.with_index do |item, index|
-            { doc_id: index, embedding: item[:embedding], document: documents[index] }
-          end
-
-          { document_embeddings: document_embeddings, dim: embeddings_with_dim.first[:dim] }
-        end
-
-        def save_vector_store(embeddings_with_config)
-          return true unless rebuild_required?
-
-          puts "Saving Vectorstore"
-          Boxcars::VectorStore::Hnswlib::SaveToHnswlib.call(
-            document_embeddings: embeddings_with_config[:document_embeddings],
-            index_file_path: index_file_path,
-            json_doc_file_path: json_doc_file_path,
-            hnswlib_config: hnswlib_config(embeddings_with_config[:dim])
-          )
-          puts "VectorStore saved"
-        end
-
-        def hnswlib_config(dim)
-          # dim: length of datum point vector that will be indexed.
-          Boxcars::VectorStore::Hnswlib::HnswlibConfig.new(
-            metric: "l2", max_item: 10000, dim: dim
-          )
-        end
-
-        def load_hnsw
-          puts "Loading Hnswlib"
-
-          config_file = "#{File.dirname(index_file_path)}/hnswlib_config.json"
-          json_config = parse_json_file(config_file)
-          document_embeddings = parse_json_file(json_doc_file_path)
-
-          search_index = ::Hnswlib::HierarchicalNSW.new(space: json_config[:metric], dim: json_config[:dim])
-          search_index.load_index(index_file_path)
-
-          { vector_store: search_index, document_embeddings: document_embeddings }
-        end
-
-        def parse_json_file(file_path)
-          return [] if file_path.nil?
-
-          file_content = File.read(file_path)
-          JSON.parse(file_content, symbolize_names: true)
-        rescue JSON::ParserError => e
-          raise_error("Error parsing hnswlib_config.json: #{e.message}")
-        end
-
-        def raise_error(message)
-          raise ::Boxcars::Error, message
-        end
-      end
-    end
-  end
-end

data/lib/boxcars/vector_store/hnswlib/hnswlib_config.rb

@@ -1,56 +0,0 @@
-# frozen_string_literal: true
-
-require 'json'
-
-module Boxcars
-  module VectorStore
-    module Hnswlib
-      class HnswlibConfig
-        attr_reader :metric, :max_item, :dim, :ef_construction, :m
-
-        # used for search index.
-        #
-        # @param max_item [Integer] The maximum number of items.
-        #
-        # @param metric [String] The distance metric between vectors ('l2', 'dot', or 'cosine').
-        #
-        # @param ef_construction [Integer] The size of the dynamic list for the nearest neighbors.
-        #                        It controls the index time/accuracy trade-off.
-        #
-        # @param max_outgoing_connection [Integer] The maximum number of outgoing connections in the graph
-        #
-        # reference: https://yoshoku.github.io/hnswlib.rb/doc/
-        def initialize(
-          metric: "l2",
-          max_item: 10000,
-          dim: 2,
-          ef_construction: 200,
-          max_outgoing_connection: 16
-        )
-          @metric = metric
-          @max_item = max_item
-          @dim = dim
-          @ef_construction = ef_construction
-          @max_outgoing_connection = max_outgoing_connection
-        end
-
-        def space
-          @metric == 'dot' ? 'ip' : 'l2'
-        end
-
-        def to_json(*args)
-          JSON.pretty_generate(
-            {
-              metric: @metric,
-              max_item: @max_item,
-              dim: @dim,
-              ef_construction: @ef_construction,
-              max_outgoing_connection: @max_outgoing_connection
-            },
-            *args
-          )
-        end
-      end
-    end
-  end
-end

data/lib/boxcars/vector_store/hnswlib/hnswlib_search.rb

@@ -1,54 +0,0 @@
-# frozen_string_literal: true
-
-require 'hnswlib'
-require 'json'
-
-module Boxcars
-  module VectorStore
-    module Hnswlib
-      class HnswlibSearch
-        def initialize(vector_store:, options: {})
-          validate_params(vector_store)
-          @vector_store = vector_store
-          @json_doc_path = options[:json_doc_path]
-          @num_neighbors = options[:num_neighbors] || 1
-        end
-
-        def call(query)
-          search(query)
-        end
-
-        private
-
-        attr_reader :json_doc_path, :vector_store, :num_neighbors
-
-        def validate_params(vector_store)
-          raise_error 'vector_store must be an Hnswlib::HierarchicalNSW' unless vector_store.is_a?(::Hnswlib::HierarchicalNSW)
-        end
-
-        def search(query)
-          raw_results = vector_store.search_knn(query, num_neighbors)
-          raw_results.map { |doc_id, distance| lookup_embedding2(doc_id, distance) }.compact
-        end
-
-        def lookup_embedding2(doc_id, distance)
-          embedding_data = parsed_data.find { |embedding| embedding[:doc_id] == doc_id }
-          return unless embedding_data
-
-          { document: embedding_data[:document], distance: distance }
-        end
-
-        def parsed_data
-          @parsed_data ||= JSON.parse(
-            File.read(json_doc_path),
-            symbolize_names: true
-          )
-        end
-
-        def raise_error(message)
-          raise ::Boxcars::ArgumentError, message
-        end
-      end
-    end
-  end
-end

data/lib/boxcars/vector_store/in_memory/add_documents.rb

@@ -1,67 +0,0 @@
-# frozen_string_literal: true
-
-module Boxcars
-  module VectorStore
-    module InMemory
-      MemoryVector = Struct.new(:content, :embedding, :metadatax)
-
-      class AddDocuments
-        include VectorStore
-
-        def initialize(embedding_tool: :openai, documents: nil)
-          validate_params(embedding_tool, documents)
-          @embedding_tool = embedding_tool
-          @documents = documents
-          @memory_vectors = []
-        end
-
-        def call
-          texts = @documents.map { |doc| doc[:page_content] }
-          vectors = generate_vectors(texts)
-          add_vectors(vectors, @documents)
-          @memory_vectors
-        end
-
-        private
-
-        def validate_params(embedding_tool, documents)
-          raise ::Boxcars::ArgumentError, 'documents is nil' unless documents
-          return if %i[openai tensorflow].include?(embedding_tool)
-
-          raise ::Boxcars::ArgumentError, 'embedding_tool is invalid'
-        end
-
-        # returns array of documents with vectors
-        def add_vectors(vectors, documents)
-          vectors.zip(documents).each do |vector, doc|
-            memory_vector = MemoryVector.new(doc[:page_content], vector, doc[:metadata])
-            @memory_vectors << memory_vector
-          end
-        end
-
-        def generate_vectors(texts)
-          embeddings_method[:klass].call(
-            texts: texts, client: embeddings_method[:client]
-          )
-        end
-
-        def embeddings_method
-          @embeddings_method ||=
-            case @embedding_tool
-            when :openai
-              { klass: Boxcars::VectorStore::EmbedViaOpenAI, client: openai_client }
-            when :tensorflow
-              { klass: Boxcars::VectorStore::EmbedViaTensorflow, client: nil }
-            end
-        end
-
-        # Get the OpenAI client
-        # @param openai_access_token [String] the OpenAI access token
-        # @return [OpenAI::Client]
-        def openai_client(openai_access_token: nil)
-          @openai_client ||= Openai.open_ai_client(openai_access_token: openai_access_token)
-        end
-      end
-    end
-  end
-end

data/lib/boxcars/vector_store/similarity_search.rb

@@ -1,55 +0,0 @@
-# frozen_string_literal: true
-
-require 'hnswlib'
-
-module Boxcars
-  module VectorStore
-    class SimilaritySearch
-      def initialize(embeddings:, vector_store:, openai_connection: nil, openai_access_token: nil)
-        @embeddings = embeddings
-        @vector_store = vector_store
-        @similarity_search_instance = create_similarity_search_instance
-        @openai_connection = openai_connection || default_connection(openai_access_token: openai_access_token)
-      end
-
-      def call(query:)
-        validate_query(query)
-        query_vector = convert_query_to_vector(query)
-        @similarity_search_instance.call(query_vector)
-      end
-
-      private
-
-      attr_reader :embeddings, :vector_store, :openai_connection
-
-      def default_connection(openai_access_token: nil)
-        Openai.open_ai_client(openai_access_token: openai_access_token)
-      end
-
-      def validate_query(query)
-        raise_error 'query must be a string' unless query.is_a?(String)
-        raise_error 'query must not be empty' if query.empty?
-      end
-
-      def convert_query_to_vector(query)
-        Boxcars::VectorStore::EmbedViaOpenAI.call(texts: [query], client: openai_connection).first[:embedding]
-      end
-
-      def create_similarity_search_instance
-        case vector_store
-        when ::Hnswlib::HierarchicalNSW
-          Boxcars::VectorStore::Hnswlib::HnswlibSearch.new(
-            vector_store: vector_store,
-            options: { json_doc_path: embeddings, num_neighbors: 2 }
-          )
-        else
-          raise_error 'Unsupported vector store provided'
-        end
-      end
-
-      def raise_error(message)
-        raise ArgumentError, message
-      end
-    end
-  end
-end