boxcars 0.2.7 → 0.2.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c58233487372c9e99f160786cc18f8e5464262479110ae2fcf2048fc09f3b338
-  data.tar.gz: 237d72649f3faef07c493e93f514077afd144578782fc8861f5a0b0b40152828
+  metadata.gz: 69b70e1d02b1ec206438eaaf857a0495fe35ab01e64a265656fe21230675306f
+  data.tar.gz: 8681b9625a0684f1091eea7a4626964929b271370068c90b82dabeee4253d803
 SHA512:
-  metadata.gz: a3c148dc072fea12862849183bfc37f4189639982e8a7409afab267aa22baed5b911dff223562705be42c5d57e06e76e6e8814b5366c120444fca529fe204187
-  data.tar.gz: acec67f2d4b5be3b660c2c6e8d90ef772925903f053f0a62965fc61f335a4fc84bbeee47265c49a596945e846d31ecad3b5a93602521037f91ed689c402712d8
+  metadata.gz: eb5c0c00f8fcdbbd6d8a1999d7544fc584c701fdf9a8a9c271fff6d9795f75ef9cab058fee2c6829808a764c892cc3e2f4e4a8717155d34d6514b46d744e632c
+  data.tar.gz: b8fb4ad34d7b93d47388f037d1d93e9e7245303740bc04d58d21942112ff97315e5dd31fdba77e275b52ddba85ca1055b897646ae1e606daee485583d50c52a6

data/CHANGELOG.md

@@ -2,7 +2,22 @@
 
 ## [Unreleased](https://github.com/BoxcarsAI/boxcars/tree/HEAD)
 
-[Full Changelog](https://github.com/BoxcarsAI/boxcars/compare/v0.2.5...HEAD)
+[Full Changelog](https://github.com/BoxcarsAI/boxcars/compare/v0.2.7...HEAD)
+
+**Closed issues:**
+
+- Getting the same verbosity as in the examples [\#54](https://github.com/BoxcarsAI/boxcars/issues/54)
+
+**Merged pull requests:**
+
+- Add Engine for Gpt4all [\#55](https://github.com/BoxcarsAI/boxcars/pull/55) ([francis](https://github.com/francis))
+- update google search to return URL for result if present [\#53](https://github.com/BoxcarsAI/boxcars/pull/53) ([francis](https://github.com/francis))
+- Draft: added gpt4all [\#49](https://github.com/BoxcarsAI/boxcars/pull/49) ([jaigouk](https://github.com/jaigouk))
+- Embeddings with hnswlib [\#48](https://github.com/BoxcarsAI/boxcars/pull/48) ([jaigouk](https://github.com/jaigouk))
+
+## [v0.2.7](https://github.com/BoxcarsAI/boxcars/tree/v0.2.7) (2023-04-13)
+
+[Full Changelog](https://github.com/BoxcarsAI/boxcars/compare/v0.2.5...v0.2.7)
 
 **Closed issues:**
 

data/Gemfile

@@ -30,3 +30,5 @@ gem "faraday-retry", "~> 2.0"
 gem "activesupport", "~> 7.0"
 
 gem "rest-client", "~> 2.1"
+
+gem "hnswlib", "~> 0.8.1"

data/Gemfile.lock

@@ -1,8 +1,9 @@
 PATH
   remote: .
   specs:
-    boxcars (0.2.7)
+    boxcars (0.2.8)
       google_search_results (~> 2.2)
+      gpt4all (~> 0.0.4)
       ruby-openai (~> 3.0)
 
 GEM
@@ -71,7 +72,12 @@ GEM
       rainbow (>= 2.2.1)
       rake (>= 10.0)
     google_search_results (2.2.0)
+    gpt4all (0.0.5)
+      faraday (~> 2.7)
+      os (~> 1.1)
+      tty-progressbar (~> 0.18.2)
     hashdiff (1.0.1)
+    hnswlib (0.8.1)
     http-accept (1.7.0)
     http-cookie (1.0.5)
       domain_name (~> 0.5)
@@ -100,6 +106,7 @@ GEM
     octokit (4.25.1)
       faraday (>= 1, < 3)
       sawyer (~> 0.9)
+    os (1.1.4)
     parallel (1.22.1)
     parser (3.2.1.1)
       ast (~> 2.4.1)
@@ -163,10 +170,19 @@ GEM
       faraday (>= 0.17.3, < 3)
     sqlite3 (1.6.2)
       mini_portile2 (~> 2.8.0)
+    sqlite3 (1.6.2-arm64-darwin)
     sqlite3 (1.6.2-x86_64-darwin)
     sqlite3 (1.6.2-x86_64-linux)
+    strings-ansi (0.2.0)
     timers (4.3.5)
     traces (0.9.1)
+    tty-cursor (0.7.1)
+    tty-progressbar (0.18.2)
+      strings-ansi (~> 0.2)
+      tty-cursor (~> 0.7)
+      tty-screen (~> 0.8)
+      unicode-display_width (>= 1.6, < 3.0)
+    tty-screen (0.8.1)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     unf (0.1.4)
@@ -181,6 +197,7 @@ GEM
       hashdiff (>= 0.4.0, < 2.0.0)
 
 PLATFORMS
+  arm64-darwin-22
   universal-java-11
   x86_64-darwin-21
   x86_64-darwin-22
@@ -194,6 +211,7 @@ DEPENDENCIES
   dotenv (~> 2.8)
   faraday-retry (~> 2.0)
   github_changelog_generator (~> 1.16)
+  hnswlib (~> 0.8.1)
   rake (~> 13.0)
   rest-client (~> 2.1)
   rspec (~> 3.2)

data/README.md

@@ -2,7 +2,6 @@
 
 <h4 align="center">
   <a href="https://www.boxcars.ai">Website</a> |
-  <a href="https://www.boxcars.ai/roadmap">Roadmap</a> |
   <a href="https://www.boxcars.ai/blog">Blog</a> |
   <a href="https://github.com/BoxcarsAI/boxcars/wiki">Documentation</a> 
 </h4>

data/boxcars.gemspec

@@ -37,6 +37,7 @@ Gem::Specification.new do |spec|
 
   # runtime dependencies
   spec.add_dependency "google_search_results", "~> 2.2"
+  spec.add_dependency "gpt4all", "~> 0.0.4"
   spec.add_dependency "ruby-openai", "~> 3.0"
 
   # For more information and examples about making a new gem, checkout our

data/lib/boxcars/boxcar/embeddings/document.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Boxcars
+  module Embeddings
+    class Document
+      attr_accessor :page_content, :metadata
+
+      def initialize(fields = {})
+        @page_content = fields[:page_content] || ""
+        @metadata = fields[:metadata] || {}
+      end
+    end
+  end
+end

data/lib/boxcars/boxcar/embeddings/embed_via_open_ai.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require 'openai'
+
+module Boxcars
+  module Embeddings
+    class EmbedViaOpenAI
+      include Embeddings
+
+      attr_accessor :texts, :openai_connection, :model
+
+      def initialize(texts:, openai_connection:, model: 'text-embedding-ada-002')
+        validate_params(texts, openai_connection)
+        @texts = texts
+        @openai_connection = openai_connection
+        @model = model
+      end
+
+      def call
+        texts.map do |text|
+          embedding = embedding_with_retry(model: model, input: strip_new_lines(text))
+          {
+            embedding: embedding,
+            dim: embedding.size
+          }
+        end
+      end
+
+      private
+
+      def validate_params(texts, openai_connection)
+        raise_error 'texts must be an array of strings' unless texts.is_a?(Array) && texts.all? { |text| text.is_a?(String) }
+        raise_error 'openai_connection must be an OpenAI::Client' unless openai_connection.is_a?(OpenAI::Client)
+      end
+
+      def embedding_with_retry(request)
+        response = @openai_connection.embeddings(parameters: request)
+        response['data'][0]['embedding']
+      end
+
+      def strip_new_lines(text)
+        text.gsub("\n", ' ')
+      end
+
+      def raise_error(message)
+        raise ::Boxcars::ValueError, message
+      end
+    end
+  end
+end

data/lib/boxcars/boxcar/embeddings/hnswlib/build_vector_store.rb

@@ -0,0 +1,159 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'hnswlib'
+require 'json'
+
+module Boxcars
+  module Embeddings
+    module Hnswlib
+      class BuildVectorStore
+        include Embeddings
+
+        # This class is responsible for building the vector store for the hnswlib similarity search.
+        # It will load the training data, generate the embeddings, and save the vector store.
+        # It will also load the vector store into memory.
+        # For later use, it will save the splitted document with index numbers to a json file.
+        #
+        # @param training_data_path [String] The path to the training data. Can be a glob pattern.
+        # @param index_file_path [String] The path to the index file.
+        # @param split_chunk_size [Integer] The number of documents to split the text into. default 2000
+        # @option json_doc_file_path [String]. The json file containing the document text.
+        #                                      if nil, it will reuse index file name.
+        # @option force_rebuild [Boolean] Optional. If true, will rebuild the index even if it already exists.
+        def initialize(
+          training_data_path:,
+          index_file_path:,
+          split_chunk_size: 2000,
+          json_doc_file_path: nil,
+          force_rebuild: true
+        )
+          @training_data_path = training_data_path
+          @index_file_path = index_file_path
+          @split_chunk_size = split_chunk_size
+          @json_doc_file_path = json_doc_file_path || index_file_path.gsub(/\.bin$/, '.json')
+          @force_rebuild = force_rebuild
+        end
+
+        def call
+          validate_params
+          data = load_files
+          documents = split_text_into_chunks(data)
+          embeddings_with_config = generate_embeddings(documents)
+          save_vector_store(embeddings_with_config)
+          load_hnsw
+        end
+
+        private
+
+        attr_reader :training_data_path, :index_file_path, :split_chunk_size, :json_doc_file_path, :force_rebuild
+
+        def validate_params
+          training_data_dir = File.dirname(training_data_path.gsub(/\*{1,2}/, ''))
+          raise_error('training_data_path parent directory must exist') unless File.directory?(training_data_dir)
+          raise_error('No files found at the training_data_path pattern') if Dir.glob(training_data_path).empty?
+
+          index_dir = File.dirname(index_file_path)
+          raise_error('index_file_path parent directory must exist') unless File.directory?(index_dir)
+
+          raise_error('split_chunk_size must be an integer') unless split_chunk_size.is_a?(Integer)
+        end
+
+        def load_files
+          data = []
+          files = Dir.glob(training_data_path)
+          raise_error "No files found at #{training_data_path}" if files.empty?
+
+          files.each do |file|
+            data << File.read(file)
+          end
+          puts "Added #{files.length} files to data. Splitting text into chunks..."
+          data
+        end
+
+        def split_text_into_chunks(data)
+          return true unless rebuild_required?
+
+          docs = []
+          data.each do |chunk|
+            doc_output = Boxcars::Embeddings::SplitText.call(
+              separator: "\n", chunk_size: split_chunk_size, chunk_overlap: 0, text: chunk
+            )
+            docs.concat(doc_output)
+          end
+          docs
+        end
+
+        def rebuild_required?
+          hnswlib_config_json = "#{File.dirname(index_file_path)}/hnswlib_config.json"
+          return true unless File.exist?(index_file_path)
+          return true if File.exist?(index_file_path) && !File.exist?(hnswlib_config_json)
+          return true if force_rebuild
+
+          false
+        end
+
+        def generate_embeddings(documents)
+          return true unless rebuild_required?
+
+          puts "Initializing Store..."
+          openai_client = OpenAI::Client.new(access_token: ENV.fetch('OPENAI_API_KEY', nil))
+
+          embeddings_with_dim = Boxcars::Embeddings::EmbedViaOpenAI.call(texts: documents, openai_connection: openai_client)
+
+          document_embeddings = embeddings_with_dim.map.with_index do |item, index|
+            { doc_id: index, embedding: item[:embedding], document: documents[index] }
+          end
+
+          { document_embeddings: document_embeddings, dim: embeddings_with_dim.first[:dim] }
+        end
+
+        def save_vector_store(embeddings_with_config)
+          return true unless rebuild_required?
+
+          puts "Saving Vectorstore"
+          Boxcars::Embeddings::Hnswlib::SaveToHnswlib.call(
+            document_embeddings: embeddings_with_config[:document_embeddings],
+            index_file_path: index_file_path,
+            json_doc_file_path: json_doc_file_path,
+            hnswlib_config: hnswlib_config(embeddings_with_config[:dim])
+          )
+          puts "VectorStore saved"
+        end
+
+        def hnswlib_config(dim)
+          # dim: length of datum point vector that will be indexed.
+          Boxcars::Embeddings::Hnswlib::HnswlibConfig.new(
+            metric: "l2", max_item: 10000, dim: dim
+          )
+        end
+
+        def load_hnsw
+          puts "Loading Hnswlib"
+
+          config_file = "#{File.dirname(index_file_path)}/hnswlib_config.json"
+          json_config = parse_json_file(config_file)
+          document_embeddings = parse_json_file(json_doc_file_path)
+
+          search_index = ::Hnswlib::HierarchicalNSW.new(space: json_config[:metric], dim: json_config[:dim])
+          search_index.load_index(index_file_path)
+
+          { vector_store: search_index, document_embeddings: document_embeddings }
+        end
+
+        def parse_json_file(file_path)
+          return [] if file_path.nil?
+
+          file_content = File.read(file_path)
+          JSON.parse(file_content, symbolize_names: true)
+        rescue JSON::ParserError => e
+          raise_error("Error parsing hnswlib_config.json: #{e.message}")
+        end
+
+        def raise_error(message)
+          raise ::Boxcars::Error, message
+        end
+      end
+    end
+  end
+end

data/lib/boxcars/boxcar/embeddings/hnswlib/hnswlib_config.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Boxcars
+  module Embeddings
+    module Hnswlib
+      class HnswlibConfig
+        attr_reader :metric, :max_item, :dim, :ef_construction, :m
+
+        # used for search index.
+        #
+        # @param max_item [Integer] The maximum number of items.
+        #
+        # @param metric [String] The distance metric between vectors ('l2', 'dot', or 'cosine').
+        #
+        # @param ef_construction [Integer] The size of the dynamic list for the nearest neighbors.
+        #                        It controls the index time/accuracy trade-off.
+        #
+        # @param max_outgoing_connection [Integer] The maximum number of outgoing connections in the graph
+        #
+        # reference: https://yoshoku.github.io/hnswlib.rb/doc/
+        def initialize(
+          metric: "l2",
+          max_item: 10000,
+          dim: 2,
+          ef_construction: 200,
+          max_outgoing_connection: 16
+        )
+          @metric = metric
+          @max_item = max_item
+          @dim = dim
+          @ef_construction = ef_construction
+          @max_outgoing_connection = max_outgoing_connection
+        end
+
+        def space
+          @metric == 'dot' ? 'ip' : 'l2'
+        end
+
+        def to_json(*args)
+          JSON.pretty_generate(
+            {
+              metric: @metric,
+              max_item: @max_item,
+              dim: @dim,
+              ef_construction: @ef_construction,
+              max_outgoing_connection: @max_outgoing_connection
+            },
+            *args
+          )
+        end
+      end
+    end
+  end
+end

data/lib/boxcars/boxcar/embeddings/hnswlib/hnswlib_search.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require 'hnswlib'
+require 'json'
+
+module Boxcars
+  module Embeddings
+    module Hnswlib
+      class HnswlibSearch
+        def initialize(vector_store:, options: {})
+          validate_params(vector_store)
+          @vector_store = vector_store
+          @json_doc_path = options[:json_doc_path]
+          @num_neighbors = options[:num_neighbors] || 1
+        end
+
+        def call(query)
+          search(query)
+        end
+
+        private
+
+        attr_reader :json_doc_path, :vector_store, :num_neighbors
+
+        def validate_params(vector_store)
+          raise_error 'vector_store must be an Hnswlib::HierarchicalNSW' unless vector_store.is_a?(::Hnswlib::HierarchicalNSW)
+        end
+
+        def search(query)
+          raw_results = vector_store.search_knn(query, num_neighbors)
+          raw_results.map { |doc_id, distance| lookup_embedding2(doc_id, distance) }.compact
+        end
+
+        def lookup_embedding2(doc_id, distance)
+          embedding_data = parsed_data.find { |embedding| embedding[:doc_id] == doc_id }
+          return unless embedding_data
+
+          { document: embedding_data[:document], distance: distance }
+        end
+
+        def parsed_data
+          @parsed_data ||= JSON.parse(
+            File.read(json_doc_path),
+            symbolize_names: true
+          )
+        end
+
+        def raise_error(message)
+          raise ArgumentError, message
+        end
+      end
+    end
+  end
+end

data/lib/boxcars/boxcar/embeddings/hnswlib/save_to_hnswlib.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require 'hnswlib'
+require 'json'
+require 'fileutils'
+
+module Boxcars
+  module Embeddings
+    module Hnswlib
+      class SaveToHnswlib
+        include Embeddings
+
+        # @param document_embeddings [Array] An array of hashes containing the document id, document text, and embedding.
+        # @param index_file_path [String] The path to the index file.
+        # @param hnswlib_config [Boxcars::Embeddings::Hnswlib::Config] The config object for the hnswlib index.
+        # @option json_doc_file_path [String] Optional. The path to the json file containing the document text.
+        def initialize(document_embeddings:, index_file_path:, hnswlib_config:, json_doc_file_path: nil)
+          @document_embeddings = document_embeddings
+          @index_file_path = index_file_path
+          @json_doc_file_path = json_doc_file_path || index_file_path.gsub(/\.bin$/, '.json')
+
+          @hnswlib_config = hnswlib_config
+          @index = ::Hnswlib::HnswIndex.new(
+            n_features: hnswlib_config.dim,
+            max_item: hnswlib_config.max_item,
+            metric: hnswlib_config.metric
+          )
+        end
+
+        def call
+          validate_params
+          document_texts = []
+
+          document_embeddings.each do |embedding|
+            index.add_item(embedding[:doc_id], embedding[:embedding])
+
+            document_texts << { doc_id: embedding[:doc_id], embedding: embedding[:embedding], document: embedding[:document] }
+          end
+
+          write_files(index, document_texts)
+        end
+
+        private
+
+        def write_files(index, document_texts)
+          FileUtils.mkdir_p(File.dirname(json_doc_file_path))
+          File.write(json_doc_file_path, document_texts.to_json)
+
+          FileUtils.mkdir_p(File.dirname(index_file_path))
+          File.write("#{File.dirname(index_file_path)}/hnswlib_config.json", hnswlib_config.to_json)
+
+          index.save(index_file_path)
+        end
+
+        attr_reader :index, :document_embeddings, :index_file_path, :json_doc_file_path, :hnswlib_config
+
+        def validate_params
+          raise_error("document_embeddings must be an array") unless document_embeddings.is_a?(Array)
+          raise_error("dim must be an integer") unless hnswlib_config.dim.is_a?(Integer)
+          raise_error("index_file_path must be a string") unless index_file_path.is_a?(String)
+
+          [index_file_path, json_doc_file_path].each do |path|
+            check_parent_directory(path)
+          end
+        end
+
+        def check_parent_directory(path)
+          return unless path
+
+          parent_dir = File.dirname(path)
+          raise_error('parent directory must exist') unless File.directory?(parent_dir)
+        end
+
+        def raise_error(message)
+          raise ::Boxcars::ValueError, message
+        end
+      end
+    end
+  end
+end

data/lib/boxcars/boxcar/embeddings/similarity_search.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require 'hnswlib'
+
+module Boxcars
+  module Embeddings
+    class SimilaritySearch
+      def initialize(embeddings:, vector_store:, openai_connection:)
+        @embeddings = embeddings
+        @vector_store = vector_store
+        @similarity_search_instance = create_similarity_search_instance
+        @openai_connection = openai_connection
+      end
+
+      def call(query:)
+        validate_query(query)
+        query_vector = convert_query_to_vector(query)
+        @similarity_search_instance.call(query_vector)
+      end
+
+      private
+
+      attr_reader :embeddings, :vector_store, :openai_connection
+
+      def validate_query(query)
+        raise_error 'query must be a string' unless query.is_a?(String)
+        raise_error 'query must not be empty' if query.empty?
+      end
+
+      def convert_query_to_vector(query)
+        Boxcars::Embeddings::EmbedViaOpenAI.call(texts: [query], openai_connection: openai_connection).first[:embedding]
+      end
+
+      def create_similarity_search_instance
+        case vector_store
+        when ::Hnswlib::HierarchicalNSW
+          Boxcars::Embeddings::Hnswlib::HnswlibSearch.new(
+            vector_store: vector_store,
+            options: { json_doc_path: embeddings, num_neighbors: 2 }
+          )
+        else
+          raise_error 'Unsupported vector store provided'
+        end
+      end
+
+      def raise_error(message)
+        raise ArgumentError, message
+      end
+    end
+  end
+end

data/lib/boxcars/boxcar/embeddings/split_text.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+module Boxcars
+  module Embeddings
+    # Split a text into chunks of a given size.
+    class SplitText
+      include Embeddings
+
+      attr_reader :separator, :chunk_size, :chunk_overlap, :text
+
+      # @param separator [String] The string to use to split the text.
+      # @param chunk_size [Integer] The size of each chunk.
+      # @param chunk_overlap [Integer] The amount of overlap between chunks.
+      # @param text [String] The text to split.
+      def initialize(separator: "Search", chunk_size: 7, chunk_overlap: 3, text: "")
+        # require 'debugger'; debugger
+        validate_params(separator, chunk_size, chunk_overlap, text)
+
+        @separator = separator
+        @chunk_size = chunk_size
+        @chunk_overlap = chunk_overlap
+        @text = text
+      end
+
+      def call
+        splits = text.split(separator)
+        merged_splits = merge_splits(splits, separator)
+
+        merged_splits&.sort
+      end
+
+      private
+
+      def validate_params(separator, chunk_size, chunk_overlap, text)
+        raise_error("separator must be a string") unless separator.is_a?(String)
+        raise_error("chunk_size must be an integer") unless chunk_size.is_a?(Integer)
+        raise_error("chunk_overlap must be an integer") unless chunk_overlap.is_a?(Integer)
+        raise_error("text must be a string") unless text.is_a?(String)
+        raise_error("chunk_overlap must be less than chunk_size") if chunk_overlap >= chunk_size
+      end
+
+      def raise_error(message)
+        raise ::Boxcars::ValueError, message
+      end
+
+      def merge_splits(splits, separator)
+        merged_splits = []
+        current_doc = []
+        total = 0
+
+        splits.each do |split|
+          split_len = split.length
+          total = process_split(total, split_len, current_doc, merged_splits, separator)
+          current_doc << split
+          total += split_len
+        end
+
+        add_remaining_doc(current_doc, merged_splits, separator)
+        merged_splits
+      end
+
+      def process_split(total, split_len, current_doc, merged_splits, separator)
+        if total + split_len >= chunk_size
+          warn_if_chunk_too_large(total)
+          total = handle_large_chunk(total, split_len, current_doc, merged_splits, separator)
+        end
+        total
+      end
+
+      def warn_if_chunk_too_large(total)
+        return unless total > chunk_size
+
+        puts "Created a chunk of size #{total}, which is longer than the specified #{chunk_size}"
+      end
+
+      def handle_large_chunk(total, split_len, current_doc, merged_splits, separator)
+        if current_doc.length.positive?
+          doc = join_docs(current_doc, separator)
+          merged_splits << doc if doc
+          total = remove_overlap(total, split_len, current_doc)
+        end
+        total
+      end
+
+      def remove_overlap(total, split_len, current_doc)
+        while total > chunk_overlap || (total + split_len > chunk_size && total.positive?)
+          total -= current_doc[0].length
+          current_doc.shift
+        end
+        total
+      end
+
+      def add_remaining_doc(current_doc, merged_splits, separator)
+        doc = join_docs(current_doc, separator)
+        merged_splits << doc if doc
+      end
+
+      def join_docs(docs, separator)
+        text = docs.join(separator).strip
+        text.empty? ? nil : text
+      end
+    end
+  end
+end

data/lib/boxcars/boxcar/embeddings.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+# Boxcars is a framework for running a series of tools to get an answer to a question.
+module Boxcars
+  module Embeddings
+    module ClassMethods
+      EmbeddingsError = Class.new(StandardError)
+
+      def call(*args, **kw_args)
+        new(*args, **kw_args).call
+      end
+    end
+
+    def self.included(base)
+      base.extend(ClassMethods)
+
+      class << base
+        private :new
+      end
+    end
+  end
+end
+
+require_relative "embeddings/document"
+require_relative "embeddings/embed_via_open_ai"
+require_relative "embeddings/split_text"
+require_relative "embeddings/similarity_search"
+require_relative "embeddings/hnswlib/hnswlib_config"
+require_relative "embeddings/hnswlib/save_to_hnswlib"
+require_relative "embeddings/hnswlib/build_vector_store"
+require_relative "embeddings/hnswlib/hnswlib_search"

data/lib/boxcars/boxcar/google_search.rb

@@ -57,7 +57,15 @@ module Boxcars
       raise Error, "Got error from SerpAPI: {res[:error]}" if res[:error]
 
       ANSWER_LOCATIONS.each do |path|
-        return res.dig(*path) if res.dig(*path)
+        next unless res.dig(*path)
+
+        Boxcars.debug("Found SERP answer at #{path}", :cyan)
+        path_link = path.dup
+        last_word = path_link.pop
+        path_link << :link
+        return { last_word => res.dig(*path), url: res.dig(*path_link) } if last_word.is_a?(Symbol) && res.dig(*path_link)
+
+        return res.dig(*path)
       end
       "No good search result found"
     end

data/lib/boxcars/embedding.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+# Boxcars is a framework for running a series of tools to get an answer to a question.
+module Boxcars
+  # For Boxcars that use an engine to do their work.
+  class Embedding < Boxcar
+    Error = Class.new(StandardError)
+  end
+end
+
+require "boxcars/boxcar/embeddings"

data/lib/boxcars/engine/gpt4all_eng.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require 'gpt4all'
+# Boxcars is a framework for running a series of tools to get an answer to a question.
+module Boxcars
+  # A engine that uses local GPT4All API.
+  class Gpt4allEng < Engine
+    attr_reader :prompts, :model_kwargs, :batch_size
+
+    # the default name of the engine
+    DEFAULT_NAME = "Gpt4all engine"
+    # the default description of the engine
+    DEFAULT_DESCRIPTION = "useful for when you need to use local AI to answer questions. " \
+                          "You should ask targeted questions"
+
+    # A engine is a container for a single tool to run.
+    # @param name [String] The name of the engine. Defaults to "OpenAI engine".
+    # @param description [String] A description of the engine. Defaults to:
+    #        useful for when you need to use AI to answer questions. You should ask targeted questions".
+    # @param prompts [Array<String>] The prompts to use when asking the engine. Defaults to [].
+    # @param batch_size [Integer] The number of prompts to send to the engine at once. Defaults to 2.
+    def initialize(name: DEFAULT_NAME, description: DEFAULT_DESCRIPTION, prompts: [], batch_size: 2, **_kwargs)
+      @prompts = prompts
+      @batch_size = batch_size
+      super(description: description, name: name)
+    end
+
+    # Get an answer from the engine.
+    # @param prompt [String] The prompt to use when asking the engine.
+    # @param openai_access_token [String] The access token to use when asking the engine.
+    #   Defaults to Boxcars.configuration.openai_access_token.
+    # @param kwargs [Hash] Additional parameters to pass to the engine if wanted.
+    def client(prompt:, inputs: {}, **_kwargs)
+      gpt4all = Gpt4all::ConversationalAI.new
+      gpt4all.prepare_resources(force_download: false)
+      gpt4all.start_bot
+      input_text = prompt.as_prompt(inputs)[:prompt]
+      Boxcars.debug("Prompt after formatting:\n#{input_text}", :cyan) if Boxcars.configuration.log_prompts
+      gpt4all.prompt(input_text)
+    rescue StandardError => e
+      Boxcars.error(["Error from gpt4all engine: #{e}", e.backtrace[-5..-1]].flatten.join("\n   "))
+    ensure
+      gpt4all.stop_bot
+    end
+
+    # get an answer from the engine for a question.
+    # @param question [String] The question to ask the engine.
+    # @param kwargs [Hash] Additional parameters to pass to the engine if wanted.
+    def run(question, **kwargs)
+      prompt = Prompt.new(template: question)
+      answer = client(prompt: prompt, **kwargs)
+      Boxcars.debug("Answer: #{answer}", :cyan)
+      answer
+    end
+  end
+end

data/lib/boxcars/engine.rb

@@ -21,3 +21,4 @@ end
 
 require "boxcars/engine/engine_result"
 require "boxcars/engine/openai"
+require "boxcars/engine/gpt4all_eng"

data/lib/boxcars/version.rb

@@ -2,5 +2,5 @@
 
 module Boxcars
   # The current version of the gem.
-  VERSION = "0.2.7"
+  VERSION = "0.2.8"
 end

data/lib/boxcars.rb

@@ -170,3 +170,4 @@ require "boxcars/ruby_repl"
 require "boxcars/engine"
 require "boxcars/boxcar"
 require "boxcars/train"
+require "boxcars/embedding"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: boxcars
 version: !ruby/object:Gem::Version
-  version: 0.2.7
+  version: 0.2.8
 platform: ruby
 authors:
 - Francis Sullivan
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-04-13 00:00:00.000000000 Z
+date: 2023-04-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: debug
@@ -67,6 +67,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.2'
+- !ruby/object:Gem::Dependency
+  name: gpt4all
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.4
 - !ruby/object:Gem::Dependency
   name: ruby-openai
   requirement: !ruby/object:Gem::Requirement
@@ -106,6 +120,15 @@ files:
 - lib/boxcars/boxcar.rb
 - lib/boxcars/boxcar/active_record.rb
 - lib/boxcars/boxcar/calculator.rb
+- lib/boxcars/boxcar/embeddings.rb
+- lib/boxcars/boxcar/embeddings/document.rb
+- lib/boxcars/boxcar/embeddings/embed_via_open_ai.rb
+- lib/boxcars/boxcar/embeddings/hnswlib/build_vector_store.rb
+- lib/boxcars/boxcar/embeddings/hnswlib/hnswlib_config.rb
+- lib/boxcars/boxcar/embeddings/hnswlib/hnswlib_search.rb
+- lib/boxcars/boxcar/embeddings/hnswlib/save_to_hnswlib.rb
+- lib/boxcars/boxcar/embeddings/similarity_search.rb
+- lib/boxcars/boxcar/embeddings/split_text.rb
 - lib/boxcars/boxcar/engine_boxcar.rb
 - lib/boxcars/boxcar/google_search.rb
 - lib/boxcars/boxcar/sql.rb
@@ -113,8 +136,10 @@ files:
 - lib/boxcars/boxcar/wikipedia_search.rb
 - lib/boxcars/conversation.rb
 - lib/boxcars/conversation_prompt.rb
+- lib/boxcars/embedding.rb
 - lib/boxcars/engine.rb
 - lib/boxcars/engine/engine_result.rb
+- lib/boxcars/engine/gpt4all_eng.rb
 - lib/boxcars/engine/openai.rb
 - lib/boxcars/generation.rb
 - lib/boxcars/prompt.rb