boxcars 0.2.12 → 0.2.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 69712266f9506d71ed3ad1fdcbfeef5a389bdbb7157d88e3f703f9a9b3ad8323
-  data.tar.gz: d7a5d0796d2963b737dc018c644042fe1e744ec7bd230f581367baf84df60f76
+  metadata.gz: 74f14f8575e4670d2be6196c5196d41dd9728b5a44a0d4e199dfb705dfc77ed5
+  data.tar.gz: 06f2e8178f9696831870b5d8d5ea40bda8ba74a2fcc27283849f49124c51a06b
 SHA512:
-  metadata.gz: 85876f5e1053bb8100795020c33da778a06668f9e3be856a8689d90d13728cef73e437ee6d5c0888b4a5483f698ee8288c061573a93fdff93559080e525c4254
-  data.tar.gz: 99e15b3fe0c5d5277c5ed123e5569bca1f1ddfca3a1b3ec054504b855bc7a005d6eb9a8f7ba71989d16ded297959fa09cddf7b31879ff37df78df5dfb21b3240
+  metadata.gz: 9ff6e759f3d942f859de85763ffe9bc0ccf5636914d894205f78cffc99abb1b27dea47463f0fc968eba6746c055f796c95884d5e421486e06374fb0519eb8c63
+  data.tar.gz: 03bf42b1fbd6dac1eff4734bd2443a4a2a9f7cb931c99a2fe3f9453f5a23f0853b4eb26c817e1757a16629617eff4704f276b99fb165689ef6016ace86c2fb56

data/Gemfile

@@ -5,6 +5,10 @@ source "https://rubygems.org"
 # Specify your gem's dependencies in boxcars.gemspec
 gemspec
 
+gem "debug", "~> 1.1"
+
+gem "dotenv", "~> 2.8"
+
 gem "rake", "~> 13.0"
 
 gem "sqlite3", "~> 1.6"

data/Gemfile.lock

@@ -1,10 +1,12 @@
 PATH
   remote: .
   specs:
-    boxcars (0.2.12)
+    boxcars (0.2.13)
       google_search_results (~> 2.2)
       gpt4all (~> 0.0.4)
-      ruby-openai (~> 4.0)
+      hnswlib (~> 0.8)
+      pgvector (~> 0.2)
+      ruby-openai (~> 4.1)
 
 GEM
   remote: https://rubygems.org/
@@ -142,7 +144,7 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
     rspec-support (3.12.0)
-    rubocop (1.50.2)
+    rubocop (1.51.0)
       json (~> 2.3)
       parallel (~> 1.10)
       parser (>= 3.2.0.0)
@@ -156,7 +158,7 @@ GEM
       parser (>= 3.2.1.0)
     rubocop-capybara (2.18.0)
       rubocop (~> 1.41)
-    rubocop-factory_bot (2.22.0)
+    rubocop-factory_bot (2.23.1)
       rubocop (~> 1.33)
     rubocop-rake (0.6.0)
       rubocop (~> 1.0)
@@ -164,7 +166,7 @@ GEM
       rubocop (~> 1.33)
       rubocop-capybara (~> 2.17)
       rubocop-factory_bot (~> 2.22)
-    ruby-openai (4.0.0)
+    ruby-openai (4.1.0)
       faraday (>= 1)
       faraday-multipart (>= 1)
     ruby-progressbar (1.13.0)
@@ -172,11 +174,11 @@ GEM
     sawyer (0.9.2)
       addressable (>= 2.3.5)
       faraday (>= 0.17.3, < 3)
-    sqlite3 (1.6.2)
+    sqlite3 (1.6.3)
       mini_portile2 (~> 2.8.0)
-    sqlite3 (1.6.2-arm64-darwin)
-    sqlite3 (1.6.2-x86_64-darwin)
-    sqlite3 (1.6.2-x86_64-linux)
+    sqlite3 (1.6.3-arm64-darwin)
+    sqlite3 (1.6.3-x86_64-darwin)
+    sqlite3 (1.6.3-x86_64-linux)
     strings-ansi (0.2.0)
     timers (4.3.5)
     traces (0.9.1)

data/boxcars.gemspec

@@ -30,15 +30,12 @@ Gem::Specification.new do |spec|
   spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  # dev / test dependencies
-  spec.add_development_dependency "debug", "~> 1.1"
-  spec.add_development_dependency "dotenv", "~> 2.8"
-  spec.add_development_dependency "rspec", "~> 3.2"
-
   # runtime dependencies
   spec.add_dependency "google_search_results", "~> 2.2"
   spec.add_dependency "gpt4all", "~> 0.0.4"
-  spec.add_dependency "ruby-openai", "~> 4.0"
+  spec.add_dependency "hnswlib", "~> 0.8"
+  spec.add_dependency "ruby-openai", "~> 4.1"
+  spec.add_dependency "pgvector", "~> 0.2"
 
   # For more information and examples about making a new gem, checkout our
   # guide at: https://bundler.io/guides/creating_gem.html

data/lib/boxcars/boxcar/active_record.rb

@@ -29,7 +29,7 @@ module Boxcars
     end
 
     # @return Hash The additional variables for this boxcar.
-    def prediction_additional
+    def prediction_additional(_inputs)
       { model_info: model_info }.merge super
     end
 

data/lib/boxcars/boxcar/engine_boxcar.rb

@@ -114,14 +114,14 @@ module Boxcars
     end
 
     # @return Hash The additional variables for this boxcar.
-    def prediction_additional
+    def prediction_additional(_inputs)
       { stop: stop, top_k: top_k }
     end
 
     # @param inputs [Hash] The inputs to the boxcar.
     # @return Hash The variables for this boxcar.
     def prediction_variables(inputs)
-      prediction_additional.merge(inputs)
+      prediction_additional(inputs).merge(inputs)
     end
 
     # remove backticks or triple backticks from the code

data/lib/boxcars/boxcar/sql.rb

@@ -26,7 +26,7 @@ module Boxcars
     end
 
     # @return Hash The additional variables for this boxcar.
-    def prediction_additional
+    def prediction_additional(_inputs)
       { schema: schema, dialect: dialect }.merge super
     end
 

data/lib/boxcars/boxcar/swagger.rb

@@ -25,7 +25,7 @@ module Boxcars
     end
 
     # @return Hash The additional variables for this boxcar.
-    def prediction_additional
+    def prediction_additional(_inputs)
       { swagger_url: swagger_url, context: context }.merge super
     end
 

data/lib/boxcars/boxcar/vector_answer.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+# Boxcars is a framework for running a series of tools to get an answer to a question.
+module Boxcars
+  # A Boxcar that interprets a prompt and executes ruby code to do math
+  class VectorAnswer < EngineBoxcar
+    # the description of this engine boxcar
+    DESC = "useful for when you need to answer questions from vector search results."
+
+    attr_reader :embeddings, :vector_documents, :search_content
+
+    # @param embeddings [Hash] The vector embeddings to use for this boxcar.
+    # @param vector_documents [Hash] The vector documents to use for this boxcar.
+    # @param engine [Boxcars::Engine] The engine to user for this boxcar. Can be inherited from a train if nil.
+    # @param prompt [Boxcars::Prompt] The prompt to use for this boxcar. Defaults to built-in prompt.
+    # @param kwargs [Hash] Any other keyword arguments to pass to the parent class.
+    def initialize(embeddings:, vector_documents:, engine: nil, prompt: nil, **kwargs)
+      the_prompt = prompt || my_prompt
+      @embeddings = embeddings
+      @vector_documents = vector_documents
+      kwargs[:stop] ||= ["```output"]
+      kwargs[:name] ||= "VectorAnswer"
+      kwargs[:description] ||= DESC
+      super(engine: engine, prompt: the_prompt, **kwargs)
+    end
+
+    # @param inputs [Hash] The inputs to use for the prediction.
+    # @return Hash The additional variables for this boxcar.
+    def prediction_additional(inputs)
+      { search_content: get_search_content(inputs[:question]) }.merge super
+    end
+
+    private
+
+    # @param results [Array] The results from the vector search.
+    # @return [String] The content of the search results.
+    def get_results_content(results)
+      results&.map do |result|
+        result[:document].content
+      end.to_a.join("\n\n")
+    end
+
+    # return the content of the search results for count results
+    # @params question [String] The question to search for.
+    # @params count [Integer] The number of results to return.
+    # @return [String] The content of the search results.
+    def get_search_content(question, count: 1)
+      search = Boxcars::VectorSearch.new(embeddings: embeddings, vector_documents: vector_documents)
+      results = search.call query: question, count: count
+      @search_content = get_search_content(results)
+    end
+
+    # our template
+    CTEMPLATE = [
+      syst("You are tasked with answering a question using these possibly relevant excerpts from a large volume of text:\n" \
+           "```text\n%<search_content>s\n```\n\n",
+           "Using the above, just answer the question as if you were answering directly."),
+      user("%<question>s")
+    ].freeze
+
+    # The prompt to use for the engine.
+    def my_prompt
+      @conversation ||= Conversation.new(lines: CTEMPLATE)
+      @my_prompt ||= ConversationPrompt.new(
+        conversation: @conversation,
+        input_variables: [:question],
+        other_inputs: [:search_content],
+        output_variables: [:answer])
+    end
+  end
+end

data/lib/boxcars/boxcar.rb

@@ -158,3 +158,4 @@ require "boxcars/boxcar/swagger"
 require "boxcars/boxcar/active_record"
 require "boxcars/vector_store"
 require "boxcars/vector_search"
+require "boxcars/boxcar/vector_answer"

data/lib/boxcars/train/zero_shot.rb

@@ -19,7 +19,7 @@ module Boxcars
     end
 
     # @return Hash The additional variables for this boxcar.
-    def prediction_additional
+    def prediction_additional(_inputs)
       { boxcar_names: boxcar_names, boxcar_descriptions: boxcar_descriptions }.merge super
     end
 

data/lib/boxcars/train.rb

@@ -69,7 +69,7 @@ module Boxcars
     # @return [Boxcars::Action] Action specifying what boxcar to use.
     def plan(intermediate_steps, **kwargs)
       thoughts = construct_scratchpad(intermediate_steps)
-      full_inputs = prediction_additional.merge(kwargs).merge(agent_scratchpad: thoughts)
+      full_inputs = prediction_additional(kwargs).merge(kwargs).merge(agent_scratchpad: thoughts)
       action = get_next_action(full_inputs)
       return TrainFinish.new({ output: action.boxcar_input }, log: action.log) if action.boxcar == finish_boxcar_name
 

data/lib/boxcars/vector_search.rb

@@ -8,7 +8,7 @@ module Boxcars
       @vector_documents = params[:vector_documents]
       @embedding_tool = params[:embedding_tool] || :openai
       @vector_search_instance = vector_search_instance
-      @openai_connection = params[:openai_connection] || default_connection(openai_access_token: openai_access_token)
+      @openai_connection = params[:openai_connection] || default_connection(openai_access_token: params[:openai_access_token])
     end
 
     def call(query:, count: 1)

data/lib/boxcars/vector_store/hnswlib/build_from_files.rb

@@ -22,7 +22,7 @@ module Boxcars
           validate_params(@training_data_path, @index_file_path, split_chunk_size)
 
           @json_doc_file_path = absolute_json_doc_file_path(@index_file_path, params[:json_doc_file_path])
-          @force_rebuild = params[:force_rebuild] || true
+          @force_rebuild = params.key?(:force_rebuild) ? params[:force_rebuild] : true
           @hnsw_vectors = []
         end
 
@@ -32,9 +32,13 @@ module Boxcars
           else
             puts "Building Hnswlib vector store..."
             data = load_data_files(training_data_path)
+            Boxcars.debug("Loaded #{data.length} files from #{training_data_path}")
             texts = split_text_into_chunks(data)
+            Boxcars.debug("Split #{data.length} files into #{texts.length} chunks")
             vectors = generate_vectors(texts)
+            Boxcars.debug("Generated #{vectors.length} vectors")
             add_vectors(vectors, texts)
+            Boxcars.debug("Added #{vectors.length} vectors to vector store")
             save_vector_store
 
             {

data/lib/boxcars/version.rb

@@ -2,5 +2,5 @@
 
 module Boxcars
   # The current version of the gem.
-  VERSION = "0.2.12"
+  VERSION = "0.2.13"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: boxcars
 version: !ruby/object:Gem::Version
-  version: 0.2.12
+  version: 0.2.13
 platform: ruby
 authors:
 - Francis Sullivan
@@ -9,92 +9,78 @@ authors:
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-05-22 00:00:00.000000000 Z
+date: 2023-05-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: debug
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.1'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.1'
-- !ruby/object:Gem::Dependency
-  name: dotenv
+  name: google_search_results
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.8'
-  type: :development
+        version: '2.2'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.8'
+        version: '2.2'
 - !ruby/object:Gem::Dependency
-  name: rspec
+  name: gpt4all
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.2'
-  type: :development
+        version: 0.0.4
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.2'
+        version: 0.0.4
 - !ruby/object:Gem::Dependency
-  name: google_search_results
+  name: hnswlib
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.2'
+        version: '0.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.2'
+        version: '0.8'
 - !ruby/object:Gem::Dependency
-  name: gpt4all
+  name: ruby-openai
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.0.4
+        version: '4.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.0.4
+        version: '4.1'
 - !ruby/object:Gem::Dependency
-  name: ruby-openai
+  name: pgvector
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '0.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '0.2'
 description: You simply set an OpenAI key, give a number of Boxcars to a Train, and
   magic ensues when you run it.
 email:
@@ -124,6 +110,7 @@ files:
 - lib/boxcars/boxcar/google_search.rb
 - lib/boxcars/boxcar/sql.rb
 - lib/boxcars/boxcar/swagger.rb
+- lib/boxcars/boxcar/vector_answer.rb
 - lib/boxcars/boxcar/wikipedia_search.rb
 - lib/boxcars/conversation.rb
 - lib/boxcars/conversation_prompt.rb
@@ -180,7 +167,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.10
+rubygems_version: 3.2.32
 signing_key:
 specification_version: 4
 summary: Boxcars is a gem that enables you to create new systems with AI composability.