boxcars 0.2.5 → 0.2.8

data/lib/boxcars/boxcar/embeddings/hnswlib/hnswlib_search.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require 'hnswlib'
+require 'json'
+
+module Boxcars
+  module Embeddings
+    module Hnswlib
+      class HnswlibSearch
+        def initialize(vector_store:, options: {})
+          validate_params(vector_store)
+          @vector_store = vector_store
+          @json_doc_path = options[:json_doc_path]
+          @num_neighbors = options[:num_neighbors] || 1
+        end
+
+        def call(query)
+          search(query)
+        end
+
+        private
+
+        attr_reader :json_doc_path, :vector_store, :num_neighbors
+
+        def validate_params(vector_store)
+          raise_error 'vector_store must be an Hnswlib::HierarchicalNSW' unless vector_store.is_a?(::Hnswlib::HierarchicalNSW)
+        end
+
+        def search(query)
+          raw_results = vector_store.search_knn(query, num_neighbors)
+          raw_results.map { |doc_id, distance| lookup_embedding2(doc_id, distance) }.compact
+        end
+
+        def lookup_embedding2(doc_id, distance)
+          embedding_data = parsed_data.find { |embedding| embedding[:doc_id] == doc_id }
+          return unless embedding_data
+
+          { document: embedding_data[:document], distance: distance }
+        end
+
+        def parsed_data
+          @parsed_data ||= JSON.parse(
+            File.read(json_doc_path),
+            symbolize_names: true
+          )
+        end
+
+        def raise_error(message)
+          raise ArgumentError, message
+        end
+      end
+    end
+  end
+end

data/lib/boxcars/boxcar/embeddings/hnswlib/save_to_hnswlib.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require 'hnswlib'
+require 'json'
+require 'fileutils'
+
+module Boxcars
+  module Embeddings
+    module Hnswlib
+      class SaveToHnswlib
+        include Embeddings
+
+        # @param document_embeddings [Array] An array of hashes containing the document id, document text, and embedding.
+        # @param index_file_path [String] The path to the index file.
+        # @param hnswlib_config [Boxcars::Embeddings::Hnswlib::Config] The config object for the hnswlib index.
+        # @option json_doc_file_path [String] Optional. The path to the json file containing the document text.
+        def initialize(document_embeddings:, index_file_path:, hnswlib_config:, json_doc_file_path: nil)
+          @document_embeddings = document_embeddings
+          @index_file_path = index_file_path
+          @json_doc_file_path = json_doc_file_path || index_file_path.gsub(/\.bin$/, '.json')
+
+          @hnswlib_config = hnswlib_config
+          @index = ::Hnswlib::HnswIndex.new(
+            n_features: hnswlib_config.dim,
+            max_item: hnswlib_config.max_item,
+            metric: hnswlib_config.metric
+          )
+        end
+
+        def call
+          validate_params
+          document_texts = []
+
+          document_embeddings.each do |embedding|
+            index.add_item(embedding[:doc_id], embedding[:embedding])
+
+            document_texts << { doc_id: embedding[:doc_id], embedding: embedding[:embedding], document: embedding[:document] }
+          end
+
+          write_files(index, document_texts)
+        end
+
+        private
+
+        def write_files(index, document_texts)
+          FileUtils.mkdir_p(File.dirname(json_doc_file_path))
+          File.write(json_doc_file_path, document_texts.to_json)
+
+          FileUtils.mkdir_p(File.dirname(index_file_path))
+          File.write("#{File.dirname(index_file_path)}/hnswlib_config.json", hnswlib_config.to_json)
+
+          index.save(index_file_path)
+        end
+
+        attr_reader :index, :document_embeddings, :index_file_path, :json_doc_file_path, :hnswlib_config
+
+        def validate_params
+          raise_error("document_embeddings must be an array") unless document_embeddings.is_a?(Array)
+          raise_error("dim must be an integer") unless hnswlib_config.dim.is_a?(Integer)
+          raise_error("index_file_path must be a string") unless index_file_path.is_a?(String)
+
+          [index_file_path, json_doc_file_path].each do |path|
+            check_parent_directory(path)
+          end
+        end
+
+        def check_parent_directory(path)
+          return unless path
+
+          parent_dir = File.dirname(path)
+          raise_error('parent directory must exist') unless File.directory?(parent_dir)
+        end
+
+        def raise_error(message)
+          raise ::Boxcars::ValueError, message
+        end
+      end
+    end
+  end
+end

data/lib/boxcars/boxcar/embeddings/similarity_search.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require 'hnswlib'
+
+module Boxcars
+  module Embeddings
+    class SimilaritySearch
+      def initialize(embeddings:, vector_store:, openai_connection:)
+        @embeddings = embeddings
+        @vector_store = vector_store
+        @similarity_search_instance = create_similarity_search_instance
+        @openai_connection = openai_connection
+      end
+
+      def call(query:)
+        validate_query(query)
+        query_vector = convert_query_to_vector(query)
+        @similarity_search_instance.call(query_vector)
+      end
+
+      private
+
+      attr_reader :embeddings, :vector_store, :openai_connection
+
+      def validate_query(query)
+        raise_error 'query must be a string' unless query.is_a?(String)
+        raise_error 'query must not be empty' if query.empty?
+      end
+
+      def convert_query_to_vector(query)
+        Boxcars::Embeddings::EmbedViaOpenAI.call(texts: [query], openai_connection: openai_connection).first[:embedding]
+      end
+
+      def create_similarity_search_instance
+        case vector_store
+        when ::Hnswlib::HierarchicalNSW
+          Boxcars::Embeddings::Hnswlib::HnswlibSearch.new(
+            vector_store: vector_store,
+            options: { json_doc_path: embeddings, num_neighbors: 2 }
+          )
+        else
+          raise_error 'Unsupported vector store provided'
+        end
+      end
+
+      def raise_error(message)
+        raise ArgumentError, message
+      end
+    end
+  end
+end

data/lib/boxcars/boxcar/embeddings/split_text.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+module Boxcars
+  module Embeddings
+    # Split a text into chunks of a given size.
+    class SplitText
+      include Embeddings
+
+      attr_reader :separator, :chunk_size, :chunk_overlap, :text
+
+      # @param separator [String] The string to use to split the text.
+      # @param chunk_size [Integer] The size of each chunk.
+      # @param chunk_overlap [Integer] The amount of overlap between chunks.
+      # @param text [String] The text to split.
+      def initialize(separator: "Search", chunk_size: 7, chunk_overlap: 3, text: "")
+        # require 'debugger'; debugger
+        validate_params(separator, chunk_size, chunk_overlap, text)
+
+        @separator = separator
+        @chunk_size = chunk_size
+        @chunk_overlap = chunk_overlap
+        @text = text
+      end
+
+      def call
+        splits = text.split(separator)
+        merged_splits = merge_splits(splits, separator)
+
+        merged_splits&.sort
+      end
+
+      private
+
+      def validate_params(separator, chunk_size, chunk_overlap, text)
+        raise_error("separator must be a string") unless separator.is_a?(String)
+        raise_error("chunk_size must be an integer") unless chunk_size.is_a?(Integer)
+        raise_error("chunk_overlap must be an integer") unless chunk_overlap.is_a?(Integer)
+        raise_error("text must be a string") unless text.is_a?(String)
+        raise_error("chunk_overlap must be less than chunk_size") if chunk_overlap >= chunk_size
+      end
+
+      def raise_error(message)
+        raise ::Boxcars::ValueError, message
+      end
+
+      def merge_splits(splits, separator)
+        merged_splits = []
+        current_doc = []
+        total = 0
+
+        splits.each do |split|
+          split_len = split.length
+          total = process_split(total, split_len, current_doc, merged_splits, separator)
+          current_doc << split
+          total += split_len
+        end
+
+        add_remaining_doc(current_doc, merged_splits, separator)
+        merged_splits
+      end
+
+      def process_split(total, split_len, current_doc, merged_splits, separator)
+        if total + split_len >= chunk_size
+          warn_if_chunk_too_large(total)
+          total = handle_large_chunk(total, split_len, current_doc, merged_splits, separator)
+        end
+        total
+      end
+
+      def warn_if_chunk_too_large(total)
+        return unless total > chunk_size
+
+        puts "Created a chunk of size #{total}, which is longer than the specified #{chunk_size}"
+      end
+
+      def handle_large_chunk(total, split_len, current_doc, merged_splits, separator)
+        if current_doc.length.positive?
+          doc = join_docs(current_doc, separator)
+          merged_splits << doc if doc
+          total = remove_overlap(total, split_len, current_doc)
+        end
+        total
+      end
+
+      def remove_overlap(total, split_len, current_doc)
+        while total > chunk_overlap || (total + split_len > chunk_size && total.positive?)
+          total -= current_doc[0].length
+          current_doc.shift
+        end
+        total
+      end
+
+      def add_remaining_doc(current_doc, merged_splits, separator)
+        doc = join_docs(current_doc, separator)
+        merged_splits << doc if doc
+      end
+
+      def join_docs(docs, separator)
+        text = docs.join(separator).strip
+        text.empty? ? nil : text
+      end
+    end
+  end
+end

data/lib/boxcars/boxcar/embeddings.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+# Boxcars is a framework for running a series of tools to get an answer to a question.
+module Boxcars
+  module Embeddings
+    module ClassMethods
+      EmbeddingsError = Class.new(StandardError)
+
+      def call(*args, **kw_args)
+        new(*args, **kw_args).call
+      end
+    end
+
+    def self.included(base)
+      base.extend(ClassMethods)
+
+      class << base
+        private :new
+      end
+    end
+  end
+end
+
+require_relative "embeddings/document"
+require_relative "embeddings/embed_via_open_ai"
+require_relative "embeddings/split_text"
+require_relative "embeddings/similarity_search"
+require_relative "embeddings/hnswlib/hnswlib_config"
+require_relative "embeddings/hnswlib/save_to_hnswlib"
+require_relative "embeddings/hnswlib/build_vector_store"
+require_relative "embeddings/hnswlib/hnswlib_search"

data/lib/boxcars/boxcar/engine_boxcar.rb

@@ -121,7 +121,7 @@ module Boxcars
     # @param inputs [Hash] The inputs to the boxcar.
     # @return Hash The variables for this boxcar.
     def prediction_variables(inputs)
-      prediction_input(inputs).merge(prediction_additional)
+      prediction_additional.merge(inputs)
     end
 
     # remove backticks or triple backticks from the code

data/lib/boxcars/boxcar/google_search.rb

@@ -34,11 +34,11 @@ module Boxcars
     # @param question [String] The question to ask Google.
     # @return [String] The location found.
     def get_location(question)
+      Boxcars.debug "Question: #{question}", :yellow
       search = ::GoogleSearch.new(q: question, limit: 3)
-      rv = search.get_location
-      puts "Question: #{question}"
-      puts "Answer: #{rv}"
-      rv
+      answer = search.get_location
+      Boxcars.debug "Answer: #{answer}", :yellow, style: :bold
+      answer
     end
 
     private
@@ -57,7 +57,15 @@ module Boxcars
       raise Error, "Got error from SerpAPI: {res[:error]}" if res[:error]
 
       ANSWER_LOCATIONS.each do |path|
-        return res.dig(*path) if res.dig(*path)
+        next unless res.dig(*path)
+
+        Boxcars.debug("Found SERP answer at #{path}", :cyan)
+        path_link = path.dup
+        last_word = path_link.pop
+        path_link << :link
+        return { last_word => res.dig(*path), url: res.dig(*path_link) } if last_word.is_a?(Symbol) && res.dig(*path_link)
+
+        return res.dig(*path)
       end
       "No good search result found"
     end

data/lib/boxcars/boxcar/sql.rb

@@ -41,6 +41,8 @@ module Boxcars
         end
       elsif rtables
         raise ArgumentError, "tables needs to be an array of Strings"
+      else
+        @requested_tables = tables
       end
       @except_models = LOCKED_OUT_TABLES + exceptions.to_a
     end
@@ -55,8 +57,8 @@ module Boxcars
        ");"].join("\n")
     end
 
-    def schema(except_tables: ['ar_internal_metadata'])
-      wanted_tables = tables.to_a - except_tables
+    def schema
+      wanted_tables = @requested_tables - @except_models
       wanted_tables.map(&method(:table_schema)).join("\n")
     end
 

data/lib/boxcars/boxcar/swagger.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+# Boxcars is a framework for running a series of tools to get an answer to a question.
+module Boxcars
+  # A Boxcar that interprets a prompt and executes API calls to get an answer.
+  class Swagger < EngineBoxcar
+    # the description of this engine boxcar
+    DESC = "useful for when you need to make Open API calls to get an answer."
+
+    attr_accessor :swagger_url, :context
+
+    # @param swagger_url [String] The URL of the Open API Swagger file to use.
+    # @param engine [Boxcars::Engine] The engine to user for this boxcar. Can be inherited from a train if nil.
+    # @param prompt [Boxcars::Prompt] The prompt to use for this boxcar. Defaults to built-in prompt.
+    # @param context [String] Additional context to use for the prompt.
+    # @param kwargs [Hash] Any other keyword arguments to pass to the parent class.
+    def initialize(swagger_url:, engine: nil, prompt: nil, context: "", **kwargs)
+      @swagger_url = swagger_url
+      @context = context
+      the_prompt = prompt || my_prompt
+      kwargs[:stop] ||= ["```output"]
+      kwargs[:name] ||= "Swagger API"
+      kwargs[:description] ||= DESC
+      super(engine: engine, prompt: the_prompt, **kwargs)
+    end
+
+    # @return Hash The additional variables for this boxcar.
+    def prediction_additional
+      { swagger_url: swagger_url, context: context }.merge super
+    end
+
+    private
+
+    def get_embedded_ruby_answer(text)
+      code = text.split("```ruby\n").last.split("```").first.strip
+      ruby_executor = Boxcars::RubyREPL.new
+      ruby_executor.call(code: code)
+    end
+
+    def get_answer(text)
+      case text
+      when /^```ruby/
+        get_embedded_ruby_answer(text)
+      when /^Answer:/
+        Result.from_text(text)
+      else
+        Result.new(status: :error,
+                   explanation: "Error: expecting your response to begin with '```ruby'. Try answering the question again.")
+      end
+    end
+
+    # our template
+    CTEMPLATE = [
+      syst("Study this Open API Swagger file %<swagger_url>s\n",
+           "and write a Ruby Program that prints the answer to the following questions using the appropriate API calls:\n",
+           "Additional context that you might need in the Ruby program: (%<context>s)\n",
+           "Use the following format:\n",
+           "${{Question needing API calls and code}}\n",
+           "reply only with the following format:\n",
+           "```ruby\n${{Ruby code with API calls and code that prints the answer}}\n```\n",
+           "```output\n${{Output of your code}}\n```\n\n",
+           "Otherwise, if you know the answer and do not need any API calls, you should use this simpler format:\n",
+           "${{Question not needing API calls}}\n",
+           "Answer: ${{Answer}}\n\n",
+           "Do not give an explanation of the answer and make sure your answer starts with either 'Answer:' or '```ruby'. ",
+           "Make use of the rest-client gem to make your requests to the API. Just print the answer."),
+      user("%<question>s")
+    ].freeze
+
+    # The prompt to use for the engine.
+    def my_prompt
+      @conversation ||= Conversation.new(lines: CTEMPLATE)
+      @my_prompt ||= ConversationPrompt.new(
+        conversation: @conversation,
+        input_variables: [:question],
+        other_inputs: [:context, :swagger_url],
+        output_variables: [:answer])
+    end
+  end
+end

data/lib/boxcars/boxcar/wikipedia_search.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "net/http"
+module Boxcars
+  # A Boxcar that uses the Wikipedia search API to get answers to questions.
+  class WikipediaSearch < Boxcar
+    # the description of this boxcar
+    WDESC = "useful for when you need to answer questions about topics from Wikipedia." \
+            "You should ask targeted questions"
+
+    # implements a boxcar that uses the Wikipedia Search to get answers to questions.
+    # @param name [String] The name of the boxcar. Defaults to classname.
+    # @param description [String] A description of the boxcar. Defaults to SERPDESC.
+    # @param serpapi_api_key [String] The API key to use for the SerpAPI. Defaults to Boxcars.configuration.serpapi_api_key.
+    def initialize(name: "Wikipedia", description: WDESC)
+      super(name: name, description: description)
+    end
+
+    # Get an answer from Google using the SerpAPI.
+    # @param question [String] The question to ask Google.
+    # @return [String] The answer to the question.
+    def run(question)
+      Boxcars.debug "Question: #{question}", :yellow
+      uri = URI("https://en.wikipedia.org/w/api.php")
+      params = { action: "query", list: "search", srsearch: question, format: "json" }
+      uri.query = URI.encode_www_form(params)
+
+      res = Net::HTTP.get_response(uri)
+      raise "Error getting response from Wikipedia: #{res.body}" unless res.is_a?(Net::HTTPSuccess)
+
+      response = JSON.parse res.body
+      answer = response.dig("query", "search", 0, "snippet").to_s.gsub(/<[^>]*>/, "")
+      pageid = response.dig("query", "search", 0, "pageid")
+      answer = "#{answer}\nurl: https://en.wikipedia.org/?curid=#{pageid}" if pageid
+      Boxcars.debug "Answer: #{answer}", :yellow, style: :bold
+      answer
+    end
+  end
+end

data/lib/boxcars/boxcar.rb

@@ -119,7 +119,10 @@ module Boxcars
 
         return run_boxcar(inputs: args[0])[output_keys.first]
       end
-      return run_boxcar(**kwargs)[output_keys].first if args.empty?
+      if args.empty?
+        ans = run_boxcar(inputs: kwargs)
+        return ans[output_keys.first]
+      end
 
       raise Boxcars::ArgumentError, "run supported with either positional or keyword arguments but not both. Got args" \
                                     ": #{args} and kwargs: #{kwargs}."
@@ -149,5 +152,7 @@ require "boxcars/result"
 require "boxcars/boxcar/engine_boxcar"
 require "boxcars/boxcar/calculator"
 require "boxcars/boxcar/google_search"
+require "boxcars/boxcar/wikipedia_search"
 require "boxcars/boxcar/sql"
+require "boxcars/boxcar/swagger"
 require "boxcars/boxcar/active_record"

data/lib/boxcars/embedding.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+# Boxcars is a framework for running a series of tools to get an answer to a question.
+module Boxcars
+  # For Boxcars that use an engine to do their work.
+  class Embedding < Boxcar
+    Error = Class.new(StandardError)
+  end
+end
+
+require "boxcars/boxcar/embeddings"

data/lib/boxcars/engine/gpt4all_eng.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require 'gpt4all'
+# Boxcars is a framework for running a series of tools to get an answer to a question.
+module Boxcars
+  # A engine that uses local GPT4All API.
+  class Gpt4allEng < Engine
+    attr_reader :prompts, :model_kwargs, :batch_size
+
+    # the default name of the engine
+    DEFAULT_NAME = "Gpt4all engine"
+    # the default description of the engine
+    DEFAULT_DESCRIPTION = "useful for when you need to use local AI to answer questions. " \
+                          "You should ask targeted questions"
+
+    # A engine is a container for a single tool to run.
+    # @param name [String] The name of the engine. Defaults to "OpenAI engine".
+    # @param description [String] A description of the engine. Defaults to:
+    #        useful for when you need to use AI to answer questions. You should ask targeted questions".
+    # @param prompts [Array<String>] The prompts to use when asking the engine. Defaults to [].
+    # @param batch_size [Integer] The number of prompts to send to the engine at once. Defaults to 2.
+    def initialize(name: DEFAULT_NAME, description: DEFAULT_DESCRIPTION, prompts: [], batch_size: 2, **_kwargs)
+      @prompts = prompts
+      @batch_size = batch_size
+      super(description: description, name: name)
+    end
+
+    # Get an answer from the engine.
+    # @param prompt [String] The prompt to use when asking the engine.
+    # @param openai_access_token [String] The access token to use when asking the engine.
+    #   Defaults to Boxcars.configuration.openai_access_token.
+    # @param kwargs [Hash] Additional parameters to pass to the engine if wanted.
+    def client(prompt:, inputs: {}, **_kwargs)
+      gpt4all = Gpt4all::ConversationalAI.new
+      gpt4all.prepare_resources(force_download: false)
+      gpt4all.start_bot
+      input_text = prompt.as_prompt(inputs)[:prompt]
+      Boxcars.debug("Prompt after formatting:\n#{input_text}", :cyan) if Boxcars.configuration.log_prompts
+      gpt4all.prompt(input_text)
+    rescue StandardError => e
+      Boxcars.error(["Error from gpt4all engine: #{e}", e.backtrace[-5..-1]].flatten.join("\n   "))
+    ensure
+      gpt4all.stop_bot
+    end
+
+    # get an answer from the engine for a question.
+    # @param question [String] The question to ask the engine.
+    # @param kwargs [Hash] Additional parameters to pass to the engine if wanted.
+    def run(question, **kwargs)
+      prompt = Prompt.new(template: question)
+      answer = client(prompt: prompt, **kwargs)
+      Boxcars.debug("Answer: #{answer}", :cyan)
+      answer
+    end
+  end
+end

data/lib/boxcars/engine.rb

@@ -21,3 +21,4 @@ end
 
 require "boxcars/engine/engine_result"
 require "boxcars/engine/openai"
+require "boxcars/engine/gpt4all_eng"

data/lib/boxcars/ruby_repl.rb

@@ -20,6 +20,7 @@ module Boxcars
         Boxcars.debug output, :red
         Result.from_error(output, code: code)
       else
+        output = ::Regexp.last_match(1) if output =~ /^\s*Answer:\s*(.*)$/m
         Boxcars.debug "Answer: #{output}", :yellow, style: :bold
         Result.from_text(output, code: code)
       end

data/lib/boxcars/version.rb

@@ -2,5 +2,5 @@
 
 module Boxcars
   # The current version of the gem.
-  VERSION = "0.2.5"
+  VERSION = "0.2.8"
 end

data/lib/boxcars.rb

@@ -170,3 +170,4 @@ require "boxcars/ruby_repl"
 require "boxcars/engine"
 require "boxcars/boxcar"
 require "boxcars/train"
+require "boxcars/embedding"