boxcars 0.2.5 → 0.2.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a3593a9df2d9d8a867729e0b6081b300125933566923ca905e2eefb16a933394
-  data.tar.gz: 90c03ea9b328b8cff10f828f8b9bf4375cce0fd574dd8a26c690776e58a58c1b
+  metadata.gz: 69b70e1d02b1ec206438eaaf857a0495fe35ab01e64a265656fe21230675306f
+  data.tar.gz: 8681b9625a0684f1091eea7a4626964929b271370068c90b82dabeee4253d803
 SHA512:
-  metadata.gz: 5de41be1f154b2c21fcd6602159a6428d8702dc318904e6e03db9de1b0ed1788b03aa10365203557b39fe268f04b4594cf6915faa941b2c64e475cd6cbb55d09
-  data.tar.gz: 2ad84e5f416b19759807d658d739d1abb8405ba6ded3b38bdf5ce9406efeadccbc2102b7b477a90df67beaa287a9810b09e59b5843090fbe4619b03500d0a4f4
+  metadata.gz: eb5c0c00f8fcdbbd6d8a1999d7544fc584c701fdf9a8a9c271fff6d9795f75ef9cab058fee2c6829808a764c892cc3e2f4e4a8717155d34d6514b46d744e632c
+  data.tar.gz: b8fb4ad34d7b93d47388f037d1d93e9e7245303740bc04d58d21942112ff97315e5dd31fdba77e275b52ddba85ca1055b897646ae1e606daee485583d50c52a6

data/CHANGELOG.md

@@ -1,5 +1,44 @@
 # Changelog
 
+## [Unreleased](https://github.com/BoxcarsAI/boxcars/tree/HEAD)
+
+[Full Changelog](https://github.com/BoxcarsAI/boxcars/compare/v0.2.7...HEAD)
+
+**Closed issues:**
+
+- Getting the same verbosity as in the examples [\#54](https://github.com/BoxcarsAI/boxcars/issues/54)
+
+**Merged pull requests:**
+
+- Add Engine for Gpt4all [\#55](https://github.com/BoxcarsAI/boxcars/pull/55) ([francis](https://github.com/francis))
+- update google search to return URL for result if present [\#53](https://github.com/BoxcarsAI/boxcars/pull/53) ([francis](https://github.com/francis))
+- Draft: added gpt4all [\#49](https://github.com/BoxcarsAI/boxcars/pull/49) ([jaigouk](https://github.com/jaigouk))
+- Embeddings with hnswlib [\#48](https://github.com/BoxcarsAI/boxcars/pull/48) ([jaigouk](https://github.com/jaigouk))
+
+## [v0.2.7](https://github.com/BoxcarsAI/boxcars/tree/v0.2.7) (2023-04-13)
+
+[Full Changelog](https://github.com/BoxcarsAI/boxcars/compare/v0.2.5...v0.2.7)
+
+**Closed issues:**
+
+- The class name in the sample code of BoxCar-Google-Search wiki has not been changed. [\#50](https://github.com/BoxcarsAI/boxcars/issues/50)
+
+**Merged pull requests:**
+
+- Add Swagger Boxcar [\#51](https://github.com/BoxcarsAI/boxcars/pull/51) ([francis](https://github.com/francis))
+- Boxcars::SQL tables and except\_tables [\#47](https://github.com/BoxcarsAI/boxcars/pull/47) ([arihh](https://github.com/arihh))
+- ActiveRecord updates and new Wikipedia Search boxcar [\#46](https://github.com/BoxcarsAI/boxcars/pull/46) ([francis](https://github.com/francis))
+- Fix README.md log\_prompts settings [\#45](https://github.com/BoxcarsAI/boxcars/pull/45) ([arihh](https://github.com/arihh))
+- Update README.md to use the GoogleSearch Boxcar [\#44](https://github.com/BoxcarsAI/boxcars/pull/44) ([stockandawe](https://github.com/stockandawe))
+
+## [v0.2.5](https://github.com/BoxcarsAI/boxcars/tree/v0.2.5) (2023-03-30)
+
+[Full Changelog](https://github.com/BoxcarsAI/boxcars/compare/v0.2.4...v0.2.5)
+
+**Merged pull requests:**
+
+- switch to safe level 4 for eval, and rerun tests [\#43](https://github.com/BoxcarsAI/boxcars/pull/43) ([francis](https://github.com/francis))
+
 ## [v0.2.4](https://github.com/BoxcarsAI/boxcars/tree/v0.2.4) (2023-03-28)
 
 [Full Changelog](https://github.com/BoxcarsAI/boxcars/compare/v0.2.3...v0.2.4)

data/Gemfile

@@ -26,3 +26,9 @@ gem "activerecord", "~> 7.0"
 gem "github_changelog_generator", "~> 1.16"
 
 gem "faraday-retry", "~> 2.0"
+
+gem "activesupport", "~> 7.0"
+
+gem "rest-client", "~> 2.1"
+
+gem "hnswlib", "~> 0.8.1"

data/Gemfile.lock

@@ -1,8 +1,9 @@
 PATH
   remote: .
   specs:
-    boxcars (0.2.5)
+    boxcars (0.2.8)
       google_search_results (~> 2.2)
+      gpt4all (~> 0.0.4)
       ruby-openai (~> 3.0)
 
 GEM
@@ -49,6 +50,8 @@ GEM
       irb (>= 1.5.0)
       reline (>= 0.3.1)
     diff-lcs (1.5.0)
+    domain_name (0.5.20190701)
+      unf (>= 0.0.5, < 1.0.0)
     dotenv (2.8.1)
     faraday (2.7.4)
       faraday-net_http (>= 2.0, < 3.1)
@@ -69,7 +72,15 @@ GEM
       rainbow (>= 2.2.1)
       rake (>= 10.0)
     google_search_results (2.2.0)
+    gpt4all (0.0.5)
+      faraday (~> 2.7)
+      os (~> 1.1)
+      tty-progressbar (~> 0.18.2)
     hashdiff (1.0.1)
+    hnswlib (0.8.1)
+    http-accept (1.7.0)
+    http-cookie (1.0.5)
+      domain_name (~> 0.5)
     httparty (0.21.0)
       mini_mime (>= 1.0.0)
       multi_xml (>= 0.5.2)
@@ -81,16 +92,21 @@ GEM
       reline (>= 0.3.0)
     json (2.6.3)
     json (2.6.3-java)
+    mime-types (3.4.1)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2023.0218.1)
     mini_mime (1.1.2)
     mini_portile2 (2.8.1)
     minitest (5.18.0)
     multi_json (1.15.0)
     multi_xml (0.6.0)
+    netrc (0.11.0)
     nio4r (2.5.8)
     nio4r (2.5.8-java)
     octokit (4.25.1)
       faraday (>= 1, < 3)
       sawyer (~> 0.9)
+    os (1.1.4)
     parallel (1.22.1)
     parser (3.2.1.1)
       ast (~> 2.4.1)
@@ -107,6 +123,11 @@ GEM
     regexp_parser (2.7.0)
     reline (0.3.3)
       io-console (~> 0.5)
+    rest-client (2.1.0)
+      http-accept (>= 1.7.0, < 2.0)
+      http-cookie (>= 1.0.2, < 2.0)
+      mime-types (>= 1.16, < 4.0)
+      netrc (~> 0.8)
     rexml (3.2.5)
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
@@ -149,12 +170,25 @@ GEM
       faraday (>= 0.17.3, < 3)
     sqlite3 (1.6.2)
       mini_portile2 (~> 2.8.0)
+    sqlite3 (1.6.2-arm64-darwin)
     sqlite3 (1.6.2-x86_64-darwin)
     sqlite3 (1.6.2-x86_64-linux)
+    strings-ansi (0.2.0)
     timers (4.3.5)
     traces (0.9.1)
+    tty-cursor (0.7.1)
+    tty-progressbar (0.18.2)
+      strings-ansi (~> 0.2)
+      tty-cursor (~> 0.7)
+      tty-screen (~> 0.8)
+      unicode-display_width (>= 1.6, < 3.0)
+    tty-screen (0.8.1)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
+    unf (0.1.4)
+      unf_ext
+    unf (0.1.4-java)
+    unf_ext (0.0.8.2)
     unicode-display_width (2.4.2)
     vcr (6.1.0)
     webmock (3.18.1)
@@ -163,6 +197,7 @@ GEM
       hashdiff (>= 0.4.0, < 2.0.0)
 
 PLATFORMS
+  arm64-darwin-22
   universal-java-11
   x86_64-darwin-21
   x86_64-darwin-22
@@ -170,12 +205,15 @@ PLATFORMS
 
 DEPENDENCIES
   activerecord (~> 7.0)
+  activesupport (~> 7.0)
   boxcars!
   debug (~> 1.1)
   dotenv (~> 2.8)
   faraday-retry (~> 2.0)
   github_changelog_generator (~> 1.16)
+  hnswlib (~> 0.8.1)
   rake (~> 13.0)
+  rest-client (~> 2.1)
   rspec (~> 3.2)
   rubocop (~> 1.21)
   rubocop-rake (~> 0.6.0)

data/README.md

@@ -2,7 +2,6 @@
 
 <h4 align="center">
   <a href="https://www.boxcars.ai">Website</a> |
-  <a href="https://www.boxcars.ai/roadmap">Roadmap</a> |
   <a href="https://www.boxcars.ai/blog">Blog</a> |
   <a href="https://github.com/BoxcarsAI/boxcars/wiki">Documentation</a> 
 </h4>
@@ -18,10 +17,10 @@ This gem was inspired by the popular Python library Langchain. However, we wante
 ## Concepts
 All of these concepts are in a module named Boxcars:
 
-- Boxcar - an encapsulation that performs something of interest (such as search, math, SQL or an Active Record Query). A Boxcar can use an Engine (described below) to do its work.
+- Boxcar - an encapsulation that performs something of interest (such as search, math, SQL, an Active Record Query, or an API call to a service). A Boxcar can use an Engine (described below) to do its work, and if not specified but needed, the default Engine is used `Boxcars.engine`.
 - Train - Given a list of Boxcars and optionally an Engine, a Train breaks down a problem into pieces for individual Boxcars to solve. The individual results are then combined until a final answer is found. ZeroShot is the only current implementation of Train (but we are adding more soon), and you can either construct it directly or use `Boxcars::train` when you want to build a Train.
-- Prompt - used by an Engine to generate text results. Most of the Boxcars have built-in prompts, so you only need to worry about these if you are extending the system.
-- Engine - an entity that generates text from a Prompt. OpenAI's LLM text generator is the default Engine if no other is specified.
+- Prompt - used by an Engine to generate text results. Our Boxcars have built-in prompts, but you have the flexibility to change or augment them if you so desire.
+- Engine - an entity that generates text from a Prompt. OpenAI's LLM text generator is the default Engine if no other is specified, and you can override the default engine if so desired (`Boxcar.configuration.default_engine`).
 
 ## Security
 Currently, our system is designed for individuals who already possess administrative privileges for their project. It is likely possible to manipulate the system's prompts to carry out malicious actions, but if you already have administrative access, you can perform such actions without requiring boxcars in the first place.
@@ -90,14 +89,16 @@ You can change the default_engine with `Boxcars::configuration.default_engine =
 
 Here is what we have so far, but please put up a PR with your new ideas.
 - GoogleSearch: uses the SERP API to do seaches
+- WikipediaSearch: uses the Wikipedia API to do searches
 - Calculator: uses an Engine to generate ruby code to do math
 - SQL: given an ActiveRecord connection, it will generate and run sql statments from a prompt.
 - ActiveRecord: given an ActiveRecord connection, it will generate and run ActiveRecord statements from a prompt.
+- Swagger: give a Swagger Open API file (YAML or JSON), answer questions about or run against the referenced service. See [here](https://github.com/BoxcarsAI/boxcars/blob/main/notebooks/swagger_examples.ipynb) for examples.
 
 ### Run a list of Boxcars
 ```ruby
 # run a Train for a calculator, and search using default Engine
-boxcars = [Boxcars::Calculator.new, Boxcars::Serp.new]
+boxcars = [Boxcars::Calculator.new, Boxcars::GoogleSearch.new]
 train = Boxcars.train.new(boxcars: boxcars)
 train.run "What is pi times the square root of the average temperature in Austin TX in January?"
 ```
@@ -105,38 +106,40 @@ Produces:
 ```text
 > Entering Zero Shot#run
 What is pi times the square root of the average temperature in Austin TX in January?
-Thought: We need to find the average temperature in Austin TX in January and then multiply it by pi and the square root of that value. We can use a search engine to find the average temperature and a calculator to perform the multiplication. 
+Thought: We need to find the average temperature in Austin TX in January and then multiply it by pi and the square root of the average temperature. We can use a search engine to find the average temperature in Austin TX in January and a calculator to perform the multiplication. 
 Question: Average temperature in Austin TX in January
-Answer: increase from 62°F to 64°F
-Observation: increase from 62°F to 64°F
-Thought: The average temperature in Austin TX in January is around 63°F.
+Answer: January Weather in Austin Texas, United States. Daily high temperatures increase by 2°F, from 62°F to 64°F, rarely falling below 45°F or exceeding 76° ...
+Observation: January Weather in Austin Texas, United States. Daily high temperatures increase by 2°F, from 62°F to 64°F, rarely falling below 45°F or exceeding 76° ...
+Thought: We have found the average temperature in Austin TX in January, which is 64°F. Now we can use a calculator to perform the multiplication.
 > Entering Calculator#run
-pi * sqrt(63)
-RubyREPL: puts(Math::PI * Math.sqrt(63))
-Answer: 24.935618646198247
+pi * sqrt(64)
+RubyREPL: puts(Math::PI * Math.sqrt(64))
+Answer: 25.132741228718345
 
-{"status":"ok","answer":"24.935618646198247","explanation":"Answer: 24.935618646198247","code":"puts(Math::PI * Math.sqrt(63))"}
+{"status":"ok","answer":"25.132741228718345","explanation":"Answer: 25.132741228718345","code":"puts(Math::PI * Math.sqrt(64))"}
 < Exiting Calculator#run
-Observation: 24.935618646198247
-The result of pi times the square root of the average temperature in Austin TX in January is approximately 24.94.
+Observation: 25.132741228718345
+We have the final answer.
 
-Final Answer: 24.94
+Final Answer: 25.132741228718345
 
 Next Actions:
 1. What is the average temperature in Austin TX in July?
-2. What is the formula for calculating the area of a circle?
-3. What is the value of pi to 10 decimal places?
+2. What is the value of pi to 10 decimal places?
+3. What is the square root of the average temperature in Miami FL in January?
 < Exiting Zero Shot#run
 ```
 ### More Examples
 See [this](https://github.com/BoxcarsAI/boxcars/blob/main/notebooks/boxcars_examples.ipynb) Jupyter Notebook for more examples.
 
+For the new Swagger boxcar, see [this](https://github.com/BoxcarsAI/boxcars/blob/main/notebooks/swagger_examples.ipynb) Jupyter Notebook.
+
 Note, some folks that we talked to didn't know that you could run Ruby Jupyter notebooks. [You can](https://github.com/SciRuby/iruby).
 
 ### Logging
 If you use this in a Rails application, or configure `Boxcars.configuration.logger = your_logger`, logging will go to your log file.
 
-Also, if you set this flag: `Boxcars.configuration.lop_prompts = true`
+Also, if you set this flag: `Boxcars.configuration.log_prompts = true`
 The actual prompts handed to the connected Engine will be logged. This is off by default because it is very wordy, but handy if you are debugging prompts.
 
 Otherwise, we print to standard out.

data/boxcars.gemspec

@@ -37,6 +37,7 @@ Gem::Specification.new do |spec|
 
   # runtime dependencies
   spec.add_dependency "google_search_results", "~> 2.2"
+  spec.add_dependency "gpt4all", "~> 0.0.4"
   spec.add_dependency "ruby-openai", "~> 3.0"
 
   # For more information and examples about making a new gem, checkout our

data/lib/boxcars/boxcar/active_record.rb

@@ -3,6 +3,7 @@
 # Boxcars is a framework for running a series of tools to get an answer to a question.
 module Boxcars
   # A Boxcar that interprets a prompt and executes SQL code to get answers
+  # rubocop:disable Metrics/ClassLength
   class ActiveRecord < EngineBoxcar
     # the description of this engine boxcar
     ARDESC = "useful for when you need to query a database for an application named %<name>s."
@@ -21,7 +22,7 @@ module Boxcars
       @approval_callback = approval_callback
       @read_only = read_only.nil? ? !approval_callback : read_only
       @code_only = kwargs.delete(:code_only) || false
-      kwargs[:name] ||= "Data"
+      kwargs[:name] ||= get_name
       kwargs[:description] ||= format(ARDESC, name: name)
       kwargs[:prompt] ||= my_prompt
       super(**kwargs)
@@ -34,6 +35,13 @@ module Boxcars
 
     private
 
+    def get_name
+      return Rails.application.class.module_parent.name if defined?(Rails)
+    rescue StandardError => e
+      boxcars.error "Error getting rails name application name: #{e.message}"
+      nil
+    end
+
     def read_only?
       read_only
     end
@@ -44,6 +52,7 @@ module Boxcars
 
     def check_models(models, exceptions)
       if models.is_a?(Array) && models.length.positive?
+        models.map { |m| m.is_a?(Class) ? m : m.constantize }
         @requested_models = models
         models.each do |m|
           raise ArgumentError, "model #{m} needs to be an Active Record model" unless m.ancestors.include?(::ActiveRecord::Base)
@@ -119,7 +128,10 @@ module Boxcars
     # @return [Object] The result of the code
     def eval_safe_wrapper(code)
       # if the code used ActiveRecord, we need to add :: in front of it to escape the module
-      new_code = code.gsub(/(\W)ActiveRecord::/, '\1::ActiveRecord::')
+      new_code = code.gsub(/\b(ActiveRecord::)/, '::\1')
+
+      # sometimes the code will have a puts or print in it, which will miss. Remove them.
+      new_code = new_code.gsub(/\b(puts|print)\b/, '')
       proc do
         $SAFE = 4
         # rubocop:disable Security/Eval
@@ -146,7 +158,11 @@ module Boxcars
     def approved?(changes_code, code)
       # find out how many changes there are
       changes = change_count(changes_code)
-      return true unless changes&.positive?
+      begin
+        return true unless changes&.positive?
+      rescue StandardError => e
+        Boscar.error "Error while computing change count: #{e.message}", :red
+      end
 
       Boxcars.debug "#{name}(Pending Changes): #{changes}", :yellow
       if read_only?
@@ -242,7 +258,7 @@ module Boxcars
            "Pay attention to use only the attribute names that you can see in the model description.\n",
            "Do not make up variable or attribute names, and do not share variables between the code in ARChanges and ARCode\n",
            "Be careful to not query for attributes that do not exist, and to use the format specified above.\n",
-           "Finally, do not use print or puts in your code."
+           "Finally, try not to use print or puts in your code"
           ),
       user("Question: %<question>s")
     ].freeze
@@ -257,4 +273,5 @@ module Boxcars
         output_variables: [:answer])
     end
   end
+  # rubocop:enable Metrics/ClassLength
 end

data/lib/boxcars/boxcar/embeddings/document.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Boxcars
+  module Embeddings
+    class Document
+      attr_accessor :page_content, :metadata
+
+      def initialize(fields = {})
+        @page_content = fields[:page_content] || ""
+        @metadata = fields[:metadata] || {}
+      end
+    end
+  end
+end

data/lib/boxcars/boxcar/embeddings/embed_via_open_ai.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require 'openai'
+
+module Boxcars
+  module Embeddings
+    class EmbedViaOpenAI
+      include Embeddings
+
+      attr_accessor :texts, :openai_connection, :model
+
+      def initialize(texts:, openai_connection:, model: 'text-embedding-ada-002')
+        validate_params(texts, openai_connection)
+        @texts = texts
+        @openai_connection = openai_connection
+        @model = model
+      end
+
+      def call
+        texts.map do |text|
+          embedding = embedding_with_retry(model: model, input: strip_new_lines(text))
+          {
+            embedding: embedding,
+            dim: embedding.size
+          }
+        end
+      end
+
+      private
+
+      def validate_params(texts, openai_connection)
+        raise_error 'texts must be an array of strings' unless texts.is_a?(Array) && texts.all? { |text| text.is_a?(String) }
+        raise_error 'openai_connection must be an OpenAI::Client' unless openai_connection.is_a?(OpenAI::Client)
+      end
+
+      def embedding_with_retry(request)
+        response = @openai_connection.embeddings(parameters: request)
+        response['data'][0]['embedding']
+      end
+
+      def strip_new_lines(text)
+        text.gsub("\n", ' ')
+      end
+
+      def raise_error(message)
+        raise ::Boxcars::ValueError, message
+      end
+    end
+  end
+end

data/lib/boxcars/boxcar/embeddings/hnswlib/build_vector_store.rb

@@ -0,0 +1,159 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'hnswlib'
+require 'json'
+
+module Boxcars
+  module Embeddings
+    module Hnswlib
+      class BuildVectorStore
+        include Embeddings
+
+        # This class is responsible for building the vector store for the hnswlib similarity search.
+        # It will load the training data, generate the embeddings, and save the vector store.
+        # It will also load the vector store into memory.
+        # For later use, it will save the splitted document with index numbers to a json file.
+        #
+        # @param training_data_path [String] The path to the training data. Can be a glob pattern.
+        # @param index_file_path [String] The path to the index file.
+        # @param split_chunk_size [Integer] The number of documents to split the text into. default 2000
+        # @option json_doc_file_path [String]. The json file containing the document text.
+        #                                      if nil, it will reuse index file name.
+        # @option force_rebuild [Boolean] Optional. If true, will rebuild the index even if it already exists.
+        def initialize(
+          training_data_path:,
+          index_file_path:,
+          split_chunk_size: 2000,
+          json_doc_file_path: nil,
+          force_rebuild: true
+        )
+          @training_data_path = training_data_path
+          @index_file_path = index_file_path
+          @split_chunk_size = split_chunk_size
+          @json_doc_file_path = json_doc_file_path || index_file_path.gsub(/\.bin$/, '.json')
+          @force_rebuild = force_rebuild
+        end
+
+        def call
+          validate_params
+          data = load_files
+          documents = split_text_into_chunks(data)
+          embeddings_with_config = generate_embeddings(documents)
+          save_vector_store(embeddings_with_config)
+          load_hnsw
+        end
+
+        private
+
+        attr_reader :training_data_path, :index_file_path, :split_chunk_size, :json_doc_file_path, :force_rebuild
+
+        def validate_params
+          training_data_dir = File.dirname(training_data_path.gsub(/\*{1,2}/, ''))
+          raise_error('training_data_path parent directory must exist') unless File.directory?(training_data_dir)
+          raise_error('No files found at the training_data_path pattern') if Dir.glob(training_data_path).empty?
+
+          index_dir = File.dirname(index_file_path)
+          raise_error('index_file_path parent directory must exist') unless File.directory?(index_dir)
+
+          raise_error('split_chunk_size must be an integer') unless split_chunk_size.is_a?(Integer)
+        end
+
+        def load_files
+          data = []
+          files = Dir.glob(training_data_path)
+          raise_error "No files found at #{training_data_path}" if files.empty?
+
+          files.each do |file|
+            data << File.read(file)
+          end
+          puts "Added #{files.length} files to data. Splitting text into chunks..."
+          data
+        end
+
+        def split_text_into_chunks(data)
+          return true unless rebuild_required?
+
+          docs = []
+          data.each do |chunk|
+            doc_output = Boxcars::Embeddings::SplitText.call(
+              separator: "\n", chunk_size: split_chunk_size, chunk_overlap: 0, text: chunk
+            )
+            docs.concat(doc_output)
+          end
+          docs
+        end
+
+        def rebuild_required?
+          hnswlib_config_json = "#{File.dirname(index_file_path)}/hnswlib_config.json"
+          return true unless File.exist?(index_file_path)
+          return true if File.exist?(index_file_path) && !File.exist?(hnswlib_config_json)
+          return true if force_rebuild
+
+          false
+        end
+
+        def generate_embeddings(documents)
+          return true unless rebuild_required?
+
+          puts "Initializing Store..."
+          openai_client = OpenAI::Client.new(access_token: ENV.fetch('OPENAI_API_KEY', nil))
+
+          embeddings_with_dim = Boxcars::Embeddings::EmbedViaOpenAI.call(texts: documents, openai_connection: openai_client)
+
+          document_embeddings = embeddings_with_dim.map.with_index do |item, index|
+            { doc_id: index, embedding: item[:embedding], document: documents[index] }
+          end
+
+          { document_embeddings: document_embeddings, dim: embeddings_with_dim.first[:dim] }
+        end
+
+        def save_vector_store(embeddings_with_config)
+          return true unless rebuild_required?
+
+          puts "Saving Vectorstore"
+          Boxcars::Embeddings::Hnswlib::SaveToHnswlib.call(
+            document_embeddings: embeddings_with_config[:document_embeddings],
+            index_file_path: index_file_path,
+            json_doc_file_path: json_doc_file_path,
+            hnswlib_config: hnswlib_config(embeddings_with_config[:dim])
+          )
+          puts "VectorStore saved"
+        end
+
+        def hnswlib_config(dim)
+          # dim: length of datum point vector that will be indexed.
+          Boxcars::Embeddings::Hnswlib::HnswlibConfig.new(
+            metric: "l2", max_item: 10000, dim: dim
+          )
+        end
+
+        def load_hnsw
+          puts "Loading Hnswlib"
+
+          config_file = "#{File.dirname(index_file_path)}/hnswlib_config.json"
+          json_config = parse_json_file(config_file)
+          document_embeddings = parse_json_file(json_doc_file_path)
+
+          search_index = ::Hnswlib::HierarchicalNSW.new(space: json_config[:metric], dim: json_config[:dim])
+          search_index.load_index(index_file_path)
+
+          { vector_store: search_index, document_embeddings: document_embeddings }
+        end
+
+        def parse_json_file(file_path)
+          return [] if file_path.nil?
+
+          file_content = File.read(file_path)
+          JSON.parse(file_content, symbolize_names: true)
+        rescue JSON::ParserError => e
+          raise_error("Error parsing hnswlib_config.json: #{e.message}")
+        end
+
+        def raise_error(message)
+          raise ::Boxcars::Error, message
+        end
+      end
+    end
+  end
+end

data/lib/boxcars/boxcar/embeddings/hnswlib/hnswlib_config.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Boxcars
+  module Embeddings
+    module Hnswlib
+      class HnswlibConfig
+        attr_reader :metric, :max_item, :dim, :ef_construction, :m
+
+        # used for search index.
+        #
+        # @param max_item [Integer] The maximum number of items.
+        #
+        # @param metric [String] The distance metric between vectors ('l2', 'dot', or 'cosine').
+        #
+        # @param ef_construction [Integer] The size of the dynamic list for the nearest neighbors.
+        #                        It controls the index time/accuracy trade-off.
+        #
+        # @param max_outgoing_connection [Integer] The maximum number of outgoing connections in the graph
+        #
+        # reference: https://yoshoku.github.io/hnswlib.rb/doc/
+        def initialize(
+          metric: "l2",
+          max_item: 10000,
+          dim: 2,
+          ef_construction: 200,
+          max_outgoing_connection: 16
+        )
+          @metric = metric
+          @max_item = max_item
+          @dim = dim
+          @ef_construction = ef_construction
+          @max_outgoing_connection = max_outgoing_connection
+        end
+
+        def space
+          @metric == 'dot' ? 'ip' : 'l2'
+        end
+
+        def to_json(*args)
+          JSON.pretty_generate(
+            {
+              metric: @metric,
+              max_item: @max_item,
+              dim: @dim,
+              ef_construction: @ef_construction,
+              max_outgoing_connection: @max_outgoing_connection
+            },
+            *args
+          )
+        end
+      end
+    end
+  end
+end