boxcars 0.2.11 → 0.2.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: da3d8b9838602151837c0cc5bb9f3cae841ba24d1c338eade82c7807d913d4bb
-  data.tar.gz: f7be434c18f0ff2c95625fe32fae25f3a5df265331425d0c1f0430ab75761578
+  metadata.gz: 69712266f9506d71ed3ad1fdcbfeef5a389bdbb7157d88e3f703f9a9b3ad8323
+  data.tar.gz: d7a5d0796d2963b737dc018c644042fe1e744ec7bd230f581367baf84df60f76
 SHA512:
-  metadata.gz: 57dd238c56f13f63a4665d4469efdabfa5f3c54f82cb6832c4158858d4b307a80c57f619633cdad6934d64186d560dcab7a62efa9adc727edfa61afbc5acc188
-  data.tar.gz: d2c782acf20c6b6b13cbfadf8f5406363b347be90a058626ec1bb21fe32baf1acb57a4a72c4770a7ad820700b465c0474a498080604e23e5d0270001d5d4aec1
+  metadata.gz: 85876f5e1053bb8100795020c33da778a06668f9e3be856a8689d90d13728cef73e437ee6d5c0888b4a5483f698ee8288c061573a93fdff93559080e525c4254
+  data.tar.gz: 99e15b3fe0c5d5277c5ed123e5569bca1f1ddfca3a1b3ec054504b855bc7a005d6eb9a8f7ba71989d16ded297959fa09cddf7b31879ff37df78df5dfb21b3240

data/.env_sample

@@ -1,2 +1,3 @@
 openai_access_token: ''
 serpapi_api_key: ''
+DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/boxcars_test'

data/.rubocop.yml

@@ -3,6 +3,7 @@ require:
   - rubocop-rake
 
 AllCops:
+  TargetRubyVersion: 3
   Exclude:
     - 'bin/{rails,rake}'
     - 'node_modules/**/*'
@@ -152,3 +153,18 @@ Style/SlicingWithRange:
 
 Bundler/OrderedGems:
   Enabled: false
+
+RSpec/MultipleMemoizedHelpers:
+  Enabled: false
+
+RSpec/PendingWithoutReason:
+  Enabled: false
+
+RSpec/NestedGroups:
+  Enabled: false
+
+RSpec/ExampleLength:
+  Enabled: false
+
+RSpec/MultipleExpectations:
+  Enabled: false

data/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## [v0.2.11](https://github.com/BoxcarsAI/boxcars/tree/v0.2.11) (2023-05-05)
+
+[Full Changelog](https://github.com/BoxcarsAI/boxcars/compare/v0.2.10...v0.2.11)
+
+**Closed issues:**
+
+- Chore: move vector store to top level [\#67](https://github.com/BoxcarsAI/boxcars/issues/67)
+
+**Merged pull requests:**
+
+- Move vector store [\#69](https://github.com/BoxcarsAI/boxcars/pull/69) ([francis](https://github.com/francis))
+
 ## [v0.2.10](https://github.com/BoxcarsAI/boxcars/tree/v0.2.10) (2023-05-05)
 
 [Full Changelog](https://github.com/BoxcarsAI/boxcars/compare/v0.2.9...v0.2.10)

data/Gemfile

@@ -7,18 +7,6 @@ gemspec
 
 gem "rake", "~> 13.0"
 
-gem "rspec", "~> 3.2"
-
-gem "rubocop", "~> 1.21"
-
-gem "vcr", "~> 6.1.0"
-
-gem "webmock", "~> 3.18.1"
-
-gem "rubocop-rake", "~> 0.6.0"
-
-gem "rubocop-rspec", "~> 2.17"
-
 gem "sqlite3", "~> 1.6"
 
 gem "activerecord", "~> 7.0"
@@ -32,3 +20,15 @@ gem "activesupport", "~> 7.0"
 gem "rest-client", "~> 2.1"
 
 gem "hnswlib", "~> 0.8.1"
+
+gem "pg", "~> 1.5", ">= 1.5.3"
+gem "pgvector", "~> 0.2.0"
+
+group :development, :test do
+  gem "rspec", "~> 3.2"
+  gem "rubocop", "~> 1.21"
+  gem "vcr", "~> 6.1.0"
+  gem "webmock", "~> 3.18.1"
+  gem "rubocop-rake", "~> 0.6.0"
+  gem "rubocop-rspec", "~> 2.17"
+end

data/Gemfile.lock

@@ -1,10 +1,10 @@
 PATH
   remote: .
   specs:
-    boxcars (0.2.11)
+    boxcars (0.2.12)
       google_search_results (~> 2.2)
       gpt4all (~> 0.0.4)
-      ruby-openai (~> 3.0)
+      ruby-openai (~> 4.0)
 
 GEM
   remote: https://rubygems.org/
@@ -19,7 +19,7 @@ GEM
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
       tzinfo (~> 2.0)
-    addressable (2.8.1)
+    addressable (2.8.4)
       public_suffix (>= 2.0.2, < 6.0)
     ast (2.4.2)
     async (1.31.0)
@@ -34,7 +34,7 @@ GEM
       protocol-http1 (~> 0.15.0)
       protocol-http2 (~> 0.15.0)
       traces (>= 0.8.0)
-    async-http-faraday (0.11.0)
+    async-http-faraday (0.12.0)
       async-http (~> 0.42)
       faraday
     async-io (1.34.3)
@@ -46,7 +46,7 @@ GEM
       fiber-local
     crack (0.4.5)
       rexml
-    debug (1.7.2)
+    debug (1.8.0)
       irb (>= 1.5.0)
       reline (>= 0.3.1)
     diff-lcs (1.5.0)
@@ -56,8 +56,10 @@ GEM
     faraday (2.7.4)
       faraday-net_http (>= 2.0, < 3.1)
       ruby2_keywords (>= 0.0.4)
-    faraday-http-cache (2.4.1)
+    faraday-http-cache (2.5.0)
       faraday (>= 0.8)
+    faraday-multipart (1.0.4)
+      multipart-post (~> 2)
     faraday-net_http (3.0.2)
     faraday-retry (2.1.0)
       faraday (~> 2.0)
@@ -81,35 +83,33 @@ GEM
     http-accept (1.7.0)
     http-cookie (1.0.5)
       domain_name (~> 0.5)
-    httparty (0.21.0)
-      mini_mime (>= 1.0.0)
-      multi_xml (>= 0.5.2)
-    i18n (1.12.0)
+    i18n (1.13.0)
       concurrent-ruby (~> 1.0)
     io-console (0.6.0)
     io-console (0.6.0-java)
-    irb (1.6.3)
+    irb (1.6.4)
       reline (>= 0.3.0)
     json (2.6.3)
     json (2.6.3-java)
     mime-types (3.4.1)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2023.0218.1)
-    mini_mime (1.1.2)
-    mini_portile2 (2.8.1)
+    mini_portile2 (2.8.2)
     minitest (5.18.0)
     multi_json (1.15.0)
-    multi_xml (0.6.0)
+    multipart-post (2.3.0)
     netrc (0.11.0)
-    nio4r (2.5.8)
-    nio4r (2.5.8-java)
+    nio4r (2.5.9)
+    nio4r (2.5.9-java)
     octokit (4.25.1)
       faraday (>= 1, < 3)
       sawyer (~> 0.9)
     os (1.1.4)
-    parallel (1.22.1)
-    parser (3.2.1.1)
+    parallel (1.23.0)
+    parser (3.2.2.1)
       ast (~> 2.4.1)
+    pg (1.5.3)
+    pgvector (0.2.0)
     protocol-hpack (1.4.2)
     protocol-http (0.24.1)
     protocol-http1 (0.15.0)
@@ -120,7 +120,7 @@ GEM
     public_suffix (5.0.1)
     rainbow (3.1.1)
     rake (13.0.6)
-    regexp_parser (2.7.0)
+    regexp_parser (2.8.0)
     reline (0.3.3)
       io-console (~> 0.5)
     rest-client (2.1.0)
@@ -133,36 +133,40 @@ GEM
       rspec-core (~> 3.12.0)
       rspec-expectations (~> 3.12.0)
       rspec-mocks (~> 3.12.0)
-    rspec-core (3.12.1)
+    rspec-core (3.12.2)
       rspec-support (~> 3.12.0)
-    rspec-expectations (3.12.2)
+    rspec-expectations (3.12.3)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
     rspec-mocks (3.12.5)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
     rspec-support (3.12.0)
-    rubocop (1.48.1)
+    rubocop (1.50.2)
       json (~> 2.3)
       parallel (~> 1.10)
       parser (>= 3.2.0.0)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.26.0, < 2.0)
+      rubocop-ast (>= 1.28.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.28.0)
+    rubocop-ast (1.28.1)
       parser (>= 3.2.1.0)
-    rubocop-capybara (2.17.1)
+    rubocop-capybara (2.18.0)
       rubocop (~> 1.41)
+    rubocop-factory_bot (2.22.0)
+      rubocop (~> 1.33)
     rubocop-rake (0.6.0)
       rubocop (~> 1.0)
-    rubocop-rspec (2.19.0)
+    rubocop-rspec (2.22.0)
       rubocop (~> 1.33)
       rubocop-capybara (~> 2.17)
-    ruby-openai (3.7.0)
-      httparty (>= 0.18.1)
+      rubocop-factory_bot (~> 2.22)
+    ruby-openai (4.0.0)
+      faraday (>= 1)
+      faraday-multipart (>= 1)
     ruby-progressbar (1.13.0)
     ruby2_keywords (0.0.5)
     sawyer (0.9.2)
@@ -212,6 +216,8 @@ DEPENDENCIES
   faraday-retry (~> 2.0)
   github_changelog_generator (~> 1.16)
   hnswlib (~> 0.8.1)
+  pg (~> 1.5, >= 1.5.3)
+  pgvector (~> 0.2.0)
   rake (~> 13.0)
   rest-client (~> 2.1)
   rspec (~> 3.2)

data/README.md

@@ -21,6 +21,7 @@ All of these concepts are in a module named Boxcars:
 - Train - Given a list of Boxcars and optionally an Engine, a Train breaks down a problem into pieces for individual Boxcars to solve. The individual results are then combined until a final answer is found. ZeroShot is the only current implementation of Train (but we are adding more soon), and you can either construct it directly or use `Boxcars::train` when you want to build a Train.
 - Prompt - used by an Engine to generate text results. Our Boxcars have built-in prompts, but you have the flexibility to change or augment them if you so desire.
 - Engine - an entity that generates text from a Prompt. OpenAI's LLM text generator is the default Engine if no other is specified, and you can override the default engine if so desired (`Boxcar.configuration.default_engine`).
+- VectorStore - a place to store and query vectors.
 
 ## Security
 Currently, our system is designed for individuals who already possess administrative privileges for their project. It is likely possible to manipulate the system's prompts to carry out malicious actions, but if you already have administrative access, you can perform such actions without requiring boxcars in the first place.
@@ -132,7 +133,9 @@ Next Actions:
 ### More Examples
 See [this](https://github.com/BoxcarsAI/boxcars/blob/main/notebooks/boxcars_examples.ipynb) Jupyter Notebook for more examples.
 
-For the new Swagger boxcar, see [this](https://github.com/BoxcarsAI/boxcars/blob/main/notebooks/swagger_examples.ipynb) Jupyter Notebook.
+For the Swagger boxcar, see [this](https://github.com/BoxcarsAI/boxcars/blob/main/notebooks/swagger_examples.ipynb) Jupyter Notebook.
+
+For simple vector storage and search, see [this](https://github.com/BoxcarsAI/boxcars/blob/main/notebooks/vector_store_examples.ipynb) Jupyter Notebook.
 
 Note, some folks that we talked to didn't know that you could run Ruby Jupyter notebooks. [You can](https://github.com/SciRuby/iruby).
 

data/boxcars.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |spec|
   spec.description = "You simply set an OpenAI key, give a number of Boxcars to a Train, and magic ensues when you run it."
   spec.homepage = "https://github.com/BoxcarsAI/boxcars"
   spec.license = "MIT"
-  spec.required_ruby_version = ">= 2.6.0"
+  spec.required_ruby_version = ">= 3.0"
 
   spec.metadata["homepage_uri"] = spec.homepage
   spec.metadata["source_code_uri"] = spec.homepage
@@ -38,7 +38,7 @@ Gem::Specification.new do |spec|
   # runtime dependencies
   spec.add_dependency "google_search_results", "~> 2.2"
   spec.add_dependency "gpt4all", "~> 0.0.4"
-  spec.add_dependency "ruby-openai", "~> 3.0"
+  spec.add_dependency "ruby-openai", "~> 4.0"
 
   # For more information and examples about making a new gem, checkout our
   # guide at: https://bundler.io/guides/creating_gem.html

data/lib/boxcars/boxcar/active_record.rb

@@ -161,7 +161,7 @@ module Boxcars
       begin
         return true unless changes&.positive?
       rescue StandardError => e
-        Boscar.error "Error while computing change count: #{e.message}", :red
+        Boxcars.error "Error while computing change count: #{e.message}", :red
       end
 
       Boxcars.debug "#{name}(Pending Changes): #{changes}", :yellow

data/lib/boxcars/boxcar.rb

@@ -156,4 +156,5 @@ require "boxcars/boxcar/wikipedia_search"
 require "boxcars/boxcar/sql"
 require "boxcars/boxcar/swagger"
 require "boxcars/boxcar/active_record"
+require "boxcars/vector_store"
 require "boxcars/vector_search"

data/lib/boxcars/engine/openai.rb

@@ -43,6 +43,10 @@ module Boxcars
       ::OpenAI::Client.new(access_token: access_token, organization_id: organization_id)
     end
 
+    def conversation_model?(model)
+      ["gpt-3.5-turbo", "gpt-4"].include?(model)
+    end
+
     # Get an answer from the engine.
     # @param prompt [String] The prompt to use when asking the engine.
     # @param openai_access_token [String] The access token to use when asking the engine.
@@ -51,7 +55,7 @@ module Boxcars
     def client(prompt:, inputs: {}, openai_access_token: nil, **kwargs)
       clnt = Openai.open_ai_client(openai_access_token: openai_access_token)
       params = open_ai_params.merge(kwargs)
-      if params[:model] == "gpt-3.5-turbo"
+      if conversation_model?(params[:model])
         prompt = prompt.first if prompt.is_a?(Array)
         params = prompt.as_messages(inputs).merge(params)
         if Boxcars.configuration.log_prompts
@@ -71,6 +75,9 @@ module Boxcars
     def run(question, **kwargs)
       prompt = Prompt.new(template: question)
       response = client(prompt: prompt, **kwargs)
+      raise Error, "OpenAI: No response from API" unless response
+      raise Error, "OpenAI: #{response['error']}" if response["error"]
+
       answer = response["choices"].map { |c| c.dig("message", "content") || c["text"] }.join("\n").strip
       puts answer
       answer

data/lib/boxcars/vector_search.rb

@@ -3,8 +3,72 @@
 # Boxcars is a framework for running a series of tools to get an answer to a question.
 module Boxcars
   # For Boxcars that use an engine to do their work.
-  class VectorSearch < Boxcar
-    Error = Class.new(StandardError)
+  class VectorSearch
+    def initialize(params)
+      @vector_documents = params[:vector_documents]
+      @embedding_tool = params[:embedding_tool] || :openai
+      @vector_search_instance = vector_search_instance
+      @openai_connection = params[:openai_connection] || default_connection(openai_access_token: openai_access_token)
+    end
+
+    def call(query:, count: 1)
+      validate_query(query)
+      query_vector = convert_query_to_vector(query)
+      @vector_search_instance.call(query_vector: query_vector, count: count)
+    end
+
+    private
+
+    attr_reader :vector_documents, :embedding_tool, :openai_connection
+
+    def vector_search_instance
+      case vector_documents[:type]
+      when :hnswlib
+        Boxcars::VectorStore::Hnswlib::Search.new(
+          vector_documents: vector_documents
+        )
+      when :in_memory
+        Boxcars::VectorStore::InMemory::Search.new(
+          vector_documents: vector_documents
+        )
+      when :pgvector
+        Boxcars::VectorStore::Pgvector::Search.new(
+          vector_documents: vector_documents
+        )
+      else
+        raise_argument_error('Unsupported vector store provided')
+      end
+    end
+
+    def default_connection(openai_access_token: nil)
+      Openai.open_ai_client(openai_access_token: openai_access_token)
+    end
+
+    def validate_query(query)
+      raise_argument_error('query must be a string') unless query.is_a?(String)
+      raise_argument_error('query must not be empty') if query.empty?
+    end
+
+    def convert_query_to_vector(query)
+      tool = embeddings_method(embedding_tool)
+      res = tool[:klass].call(
+        texts: [query], client: tool[:client]
+      ).first
+      res[:embedding]
+    end
+
+    def embeddings_method(embedding_tool)
+      case embedding_tool
+      when :openai
+        { klass: Boxcars::VectorStore::EmbedViaOpenAI, client: openai_connection }
+      when :tensorflow
+        { klass: Boxcars::VectorStore::EmbedViaTensorflow, client: nil }
+      end
+    end
+
+    def raise_argument_error(message)
+      raise ::Boxcars::ArgumentError, message
+    end
   end
 end
 

data/lib/boxcars/vector_store/document.rb

@@ -3,10 +3,11 @@
 module Boxcars
   module VectorStore
     class Document
-      attr_accessor :page_content, :metadata
+      attr_accessor :content, :metadata, :embedding
 
       def initialize(fields = {})
-        @page_content = fields[:page_content] || ""
+        @content = fields[:content] || ""
+        @embedding = fields[:embedding] || []
         @metadata = fields[:metadata] || {}
       end
     end

data/lib/boxcars/vector_store/embed_via_open_ai.rb

@@ -7,8 +7,6 @@ module Boxcars
     class EmbedViaOpenAI
       include VectorStore
 
-      attr_accessor :texts, :client, :model
-
       def initialize(texts:, client:, model: 'text-embedding-ada-002')
         validate_params(texts, client)
         @texts = texts
@@ -28,6 +26,8 @@ module Boxcars
 
       private
 
+      attr_accessor :texts, :client, :model
+
       def validate_params(texts, client)
         raise_error 'texts must be an array of strings' unless texts.is_a?(Array) && texts.all? { |text| text.is_a?(String) }
         raise_error 'openai_connection must be an OpenAI::Client' unless client.is_a?(OpenAI::Client)

data/lib/boxcars/vector_store/hnswlib/build_from_files.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'hnswlib'
+require 'json'
+
+module Boxcars
+  module VectorStore
+    module Hnswlib
+      # This class is responsible for building the vector store for the hnswlib similarity search.
+      # It will load the training data, generate the embeddings, and save the vector store.
+      # It will also load the vector store into memory.
+      # For later use, it will save the splitted document with index numbers to a json file.
+      class BuildFromFiles
+        include VectorStore
+
+        def initialize(params)
+          @split_chunk_size = params[:split_chunk_size] || 2000
+          @training_data_path = File.absolute_path(params[:training_data_path])
+          @index_file_path = File.absolute_path(params[:index_file_path])
+
+          validate_params(@training_data_path, @index_file_path, split_chunk_size)
+
+          @json_doc_file_path = absolute_json_doc_file_path(@index_file_path, params[:json_doc_file_path])
+          @force_rebuild = params[:force_rebuild] || true
+          @hnsw_vectors = []
+        end
+
+        def call
+          if !force_rebuild && File.exist?(index_file_path)
+            load_existing_vector_store
+          else
+            puts "Building Hnswlib vector store..."
+            data = load_data_files(training_data_path)
+            texts = split_text_into_chunks(data)
+            vectors = generate_vectors(texts)
+            add_vectors(vectors, texts)
+            save_vector_store
+
+            {
+              type: :hnswlib,
+              vector_store: hnsw_vectors
+            }
+          end
+        end
+
+        private
+
+        attr_reader :training_data_path, :index_file_path, :split_chunk_size, :json_doc_file_path, :force_rebuild, :hnsw_vectors
+
+        def validate_params(training_data_path, index_file_path, split_chunk_size)
+          training_data_dir = File.dirname(training_data_path.gsub(/\*{1,2}/, ''))
+
+          raise_argument_error('training_data_path parent directory must exist') unless File.directory?(training_data_dir)
+          raise_argument_error('No files found at the training_data_path pattern') if Dir.glob(training_data_path).empty?
+
+          index_dir = File.dirname(index_file_path)
+
+          raise_argument_error('index_file_path parent directory must exist') unless File.directory?(index_dir)
+          raise_argument_error('split_chunk_size must be an integer') unless split_chunk_size.is_a?(Integer)
+        end
+
+        def absolute_json_doc_file_path(index_file_path, json_doc_file_path)
+          return index_file_path.gsub(/\.bin$/, '.json') unless json_doc_file_path
+
+          File.absolute_path(json_doc_file_path)
+        end
+
+        def add_vectors(vectors, texts)
+          vectors.map.with_index do |vector, index|
+            hnsw_vector = Document.new(
+              content: texts[index],
+              embedding: vector[:embedding],
+              metadata: {
+                doc_id: index,
+                dim: vector[:dim],
+                metric: 'l2',
+                max_item: 10000,
+                index_file_path: index_file_path,
+                json_doc_file_path: json_doc_file_path
+              }
+            )
+            hnsw_vectors << hnsw_vector
+          end
+        end
+
+        def save_vector_store
+          Boxcars::VectorStore::Hnswlib::SaveToHnswlib.call(hnsw_vectors)
+        end
+
+        def load_existing_vector_store
+          Boxcars::VectorStore::Hnswlib::LoadFromDisk.call(
+            index_file_path: index_file_path,
+            json_doc_file_path: json_doc_file_path
+          )
+        end
+      end
+    end
+  end
+end

data/lib/boxcars/vector_store/hnswlib/load_from_disk.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'hnswlib'
+require 'json'
+
+module Boxcars
+  module VectorStore
+    module Hnswlib
+      class LoadFromDisk
+        include VectorStore
+
+        def initialize(params)
+          validate_params(params[:index_file_path], params[:json_doc_file_path])
+
+          @index_file_path = File.absolute_path(params[:index_file_path])
+          @json_doc_file_path = File.absolute_path(params[:json_doc_file_path])
+        end
+
+        def call
+          vectors = parse_json_file(json_doc_file_path)
+          hnsw_vectors = load_as_hnsw_vectors(vectors)
+
+          {
+            type: :hnswlib,
+            vector_store: hnsw_vectors
+          }
+        end
+
+        private
+
+        attr_reader :index_file_path, :json_doc_file_path
+
+        def validate_params(index_file_path, json_doc_file_path)
+          raise_argument_error("index_file_path must be a string") unless index_file_path.is_a?(String)
+          raise_argument_error("json_doc_file_path must be a string") unless json_doc_file_path.is_a?(String)
+
+          raise_argument_error("index_file_path must exist") unless File.exist?(index_file_path)
+          raise_argument_error("json_doc_file_path must exist") unless File.exist?(json_doc_file_path)
+        end
+
+        def load_as_hnsw_vectors(vectors)
+          hnsw_vectors = []
+          vectors.each do |vector|
+            hnsw_vector = Document.new(
+              content: vector[:document],
+              embedding: vector[:embedding],
+              metadata: vector[:metadata]
+            )
+            hnsw_vectors[vectors.first[:doc_id].to_i] = hnsw_vector
+          end
+          hnsw_vectors
+        end
+      end
+    end
+  end
+end