boxcars 0.2.0 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 34001190ca60f2b9fe68d603267ffccaaef398cae2f8054e34d6d0926fd8bf9d
-  data.tar.gz: 7918d0bad50e6758490a9baa58393e38071e524b0d284c43dd479cf1d091deab
+  metadata.gz: 967f04279864f1419fbe3e9f02817cc8f5594317c6f2b06dd49d1c5e1baa67a9
+  data.tar.gz: 156ca722488b91cefce4d03026490434004e1dcbd1f7c7cd714f1d3f6871cbc0
 SHA512:
-  metadata.gz: dacd721802478a783ac9c50c4ebd959764b7601254c3a4b94546f093177cd94bdb4636d9f635154e6419a7d8505152ac7092c491238baab8ac0e47277323a7af
-  data.tar.gz: 73c638b8486681e3268ff94e390550f565c011891e9c025556f41df4e0e741ae705fa9fdeb6c3d1996919344abbfff84a8f1454e22a5d108dcae6d65e4718403
+  metadata.gz: 14158dbd32be6d95e35c2331e26d2e8883bc7621585d5f24b54d1fce6ba78b15e3d67df73b9ed13faf1fd5c306b9341cd4705a3ecf9e5beaf9d47f05e7c1f438
+  data.tar.gz: 90722b9f3b7f34607cd4ee4afe5888c625c5724904943a5160021b48ca1944af5a87269613aca554b595c73c32c923ee9fc442eb8b0271a712fe6aac5702f2ac

data/CHANGELOG.md

@@ -1,5 +1,51 @@
 # Changelog
 
+## [Unreleased](https://github.com/BoxcarsAI/boxcars/tree/HEAD)
+
+[Full Changelog](https://github.com/BoxcarsAI/boxcars/compare/v0.2.1...HEAD)
+
+**Implemented enhancements:**
+
+- return a structure from boxcars instead of just a string [\#31](https://github.com/BoxcarsAI/boxcars/issues/31)
+- modify SQL boxcar to take a list of Tables. Handy if you have a large database with many tables. [\#19](https://github.com/BoxcarsAI/boxcars/issues/19)
+
+**Closed issues:**
+
+- make a new type of prompt that uses conversations [\#38](https://github.com/BoxcarsAI/boxcars/issues/38)
+- Add test coverage [\#8](https://github.com/BoxcarsAI/boxcars/issues/8)
+
+## [v0.2.1](https://github.com/BoxcarsAI/boxcars/tree/v0.2.1) (2023-03-08)
+
+[Full Changelog](https://github.com/BoxcarsAI/boxcars/compare/v0.2.0...v0.2.1)
+
+**Implemented enhancements:**
+
+- add the ability to use the ChatGPT API - a ChatGPT Boxcar::Engine [\#32](https://github.com/BoxcarsAI/boxcars/issues/32)
+- Prompt simplification [\#29](https://github.com/BoxcarsAI/boxcars/issues/29)
+
+**Merged pull requests:**
+
+- use structured results internally and bring SQL up to parity with ActiveRecord boxcar. [\#36](https://github.com/BoxcarsAI/boxcars/pull/36) ([francis](https://github.com/francis))
+
+## [v0.2.0](https://github.com/BoxcarsAI/boxcars/tree/v0.2.0) (2023-03-07)
+
+[Full Changelog](https://github.com/BoxcarsAI/boxcars/compare/v0.1.8...v0.2.0)
+
+**Merged pull requests:**
+
+- Default to chatgpt [\#35](https://github.com/BoxcarsAI/boxcars/pull/35) ([francis](https://github.com/francis))
+
+## [v0.1.8](https://github.com/BoxcarsAI/boxcars/tree/v0.1.8) (2023-03-02)
+
+[Full Changelog](https://github.com/BoxcarsAI/boxcars/compare/v0.1.7...v0.1.8)
+
+**Merged pull requests:**
+
+- return JSON from the Active Record boxcar [\#34](https://github.com/BoxcarsAI/boxcars/pull/34) ([francis](https://github.com/francis))
+- validate return values from Open AI API [\#33](https://github.com/BoxcarsAI/boxcars/pull/33) ([francis](https://github.com/francis))
+- simplify prompting and parameters used. refs \#29 [\#30](https://github.com/BoxcarsAI/boxcars/pull/30) ([francis](https://github.com/francis))
+- \[infra\] Added sample .env file and updated the lookup to save the key [\#27](https://github.com/BoxcarsAI/boxcars/pull/27) ([AKovtunov](https://github.com/AKovtunov))
+
 ## [v0.1.7](https://github.com/BoxcarsAI/boxcars/tree/v0.1.7) (2023-02-27)
 
 [Full Changelog](https://github.com/BoxcarsAI/boxcars/compare/v0.1.6...v0.1.7)

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    boxcars (0.2.0)
+    boxcars (0.2.2)
       google_search_results (~> 2.2)
       ruby-openai (~> 3.0)
 

data/lib/boxcars/boxcar/active_record.rb

@@ -7,28 +7,24 @@ module Boxcars
     # the description of this engine boxcar
     ARDESC = "useful for when you need to query a database for an application named %<name>s."
     LOCKED_OUT_MODELS = %w[ActiveRecord::SchemaMigration ActiveRecord::InternalMetadata ApplicationRecord].freeze
-    attr_accessor :connection, :requested_models, :read_only, :approval_callback
+    attr_accessor :connection, :requested_models, :read_only, :approval_callback, :code_only
     attr_reader :except_models
 
-    # @param engine [Boxcars::Engine] The engine to user for this boxcar. Can be inherited from a train if nil.
     # @param models [Array<ActiveRecord::Model>] The models to use for this boxcar. Will use all if nil.
+    # @param except_models [Array<ActiveRecord::Model>] The models to exclude from this boxcar. Will exclude none if nil.
     # @param read_only [Boolean] Whether to use read only models. Defaults to true unless you pass an approval function.
     # @param approval_callback [Proc] A function to call to approve changes. Defaults to nil.
     # @param kwargs [Hash] Any other keyword arguments. These can include:
-    #   :name, :description, :prompt, :except_models, :top_k, and :stop
-    def initialize(engine: nil, models: nil, read_only: nil, approval_callback: nil, **kwargs)
-      check_models(models)
-      @except_models = LOCKED_OUT_MODELS + kwargs[:except_models].to_a
+    #   :name, :description, :prompt, :except_models, :top_k, :stop, :code_only and :engine
+    def initialize(models: nil, except_models: nil, read_only: nil, approval_callback: nil, **kwargs)
+      check_models(models, except_models)
       @approval_callback = approval_callback
       @read_only = read_only.nil? ? !approval_callback : read_only
-      the_prompt = kwargs[prompt] || my_prompt
-      name = kwargs[:name] || "Data"
-      kwargs[:stop] ||= ["Answer:"]
-      super(name: name,
-            description: kwargs[:description] || format(ARDESC, name: name),
-            engine: engine,
-            prompt: the_prompt,
-            **kwargs)
+      @code_only = kwargs.delete(:code_only) || false
+      kwargs[:name] ||= "Data"
+      kwargs[:description] ||= format(ARDESC, name: name)
+      kwargs[:prompt] ||= my_prompt
+      super(**kwargs)
     end
 
     # @return Hash The additional variables for this boxcar.
@@ -42,7 +38,11 @@ module Boxcars
       read_only
     end
 
-    def check_models(models)
+    def code_only?
+      code_only
+    end
+
+    def check_models(models, exceptions)
       if models.is_a?(Array) && models.length.positive?
         @requested_models = models
         models.each do |m|
@@ -51,6 +51,7 @@ module Boxcars
       elsif models
         raise ArgumentError, "models needs to be an array of Active Record models"
       end
+      @except_models = LOCKED_OUT_MODELS + exceptions.to_a
     end
 
     def wanted_models
@@ -120,7 +121,7 @@ module Boxcars
       changes = change_count(changes_code)
       return true unless changes&.positive?
 
-      Boxcars.debug "Pending Changes: #{changes}", :yellow, style: :bold
+      Boxcars.debug "#{name}(Pending Changes): #{changes}", :yellow
       change_str = "#{changes} change#{'s' if changes.to_i > 1}"
       raise SecurityError, "Can not run code that makes #{change_str} in read-only mode" if read_only?
 
@@ -130,6 +131,7 @@ module Boxcars
     end
 
     def run_active_record_code(code)
+      code = ::Regexp.last_match(1) if code =~ /`(.+)`/
       Boxcars.debug code, :yellow
       if read_only?
         rollback_after_running do
@@ -140,18 +142,39 @@ module Boxcars
       end
     end
 
-    def get_active_record_answer(text)
-      code = text[/^ARCode: (.*)/, 1]
-      changes_code = text[/^ARChanges: (.*)/, 1]
-      raise SecurityError, "Permission to run code that makes changes denied" unless approved?(changes_code, code)
-
-      output = run_active_record_code(code)
+    def clean_up_output(output)
+      output = output.as_json if output.is_a?(::ActiveRecord::Result)
       output = 0 if output.is_a?(Array) && output.empty?
       output = output.first if output.is_a?(Array) && output.length == 1
       output = output[output.keys.first] if output.is_a?(Hash) && output.length == 1
-      "Answer: #{output.to_json}"
-    rescue StandardError => e
-      "Error: #{e.message}"
+      output = output.as_json if output.is_a?(::ActiveRecord::Relation)
+      output
+    end
+
+    def extract_code(code)
+      case code
+      when /^```ruby/
+        code.split('```ruby').last.split('```').first.strip
+      when /^`(.+)`/
+        ::Regexp.last_match(1)
+      else
+        code
+      end
+    end
+
+    def get_active_record_answer(text)
+      code = extract_code text.split('ARChanges:').first.strip.split('ARCode:').last.strip
+      changes_code = extract_code text.split('ARChanges:').last.strip if text =~ /^ARChanges:/
+      return Result.new(status: :ok, explanation: "code to run", code: code, changes_code: changes_code) if code_only?
+
+      raise SecurityError, "Permission to run code that makes changes denied" unless approved?(changes_code, code)
+
+      output = clean_up_output(run_active_record_code(code))
+      Result.new(status: :ok, answer: output, explanation: "Answer: #{output.to_json}", code: code)
+    rescue SecurityError, ConfigurationError => e
+      raise e
+    rescue Boxcars::Error => e
+      Result.new(status: :error, answer: nil, explanation: "Error: #{e.message}", code: code)
     end
 
     def get_answer(text)
@@ -159,39 +182,42 @@ module Boxcars
       when /^ARCode:/
         get_active_record_answer(text)
       when /^Answer:/
-        text
+        Result.from_text(text)
       else
-        raise Boxcars::Error "Unknown format from engine: #{text}"
+        Result.from_error("Try answering again. Expected your answer to start with 'ARCode:'. You gave me:\n#{text}")
       end
     end
 
-    TEMPLATE = <<~IPT
-      Given an input question, first create a syntactically correct Rails Active Record code to run,
-      then look at the results of the code and return the answer. Unless the user specifies
-      in her question a specific number of examples she wishes to obtain, limit your code
-      to at most %<top_k>s results.
-
-      Never query for all the columns from a specific model, only ask for a the few relevant attributes given the question.
-
-      Pay attention to use only the attribute names that you can see in the model description. Be careful to not query for attributes that do not exist.
-      Also, pay attention to which attribute is in which model.
-
-      Use the following format:
-      Question: "Question here"
-      ARCode: "Active Record code to run"
-      ARChanges: "Active Record code to compute the number of records going to change" - Only add this line if the ARCode on the line before will make data changes
-      Answer: "Final answer here"
-
-      Only use the following Active Record models:
-      %<model_info>s
-
-      Question: %<question>s
-    IPT
+    CTEMPLATE = [
+      syst("You are a Ruby on Rails Active Record code generator"),
+      syst("Given an input question, first create a syntactically correct Rails Active Record code to run, ",
+           "then look at the results of the code and return the answer. Unless the user specifies ",
+           "in her question a specific number of examples she wishes to obtain, limit your code ",
+           "to at most %<top_k>s results.\n",
+           "Never query for all the columns from a specific model, ",
+           "only ask for the relevant attributes given the question.\n",
+           "Also, pay attention to which attribute is in which model."),
+      syst("Use the following format:\n",
+           "Question: ${{Question here}}\n",
+           "ARCode: ${{Active Record code to run}} - make sure you use valid code\n",
+           "ARChanges: ${{Active Record code to compute the number of records going to change}} - ",
+           "Only add this line if the ARCode on the line before will make data changes.\n",
+           "Answer: ${{Final answer here}}"),
+      syst("Only use the following Active Record models: %<model_info>s\n",
+           "Pay attention to use only the attribute names that you can see in the model description. ",
+           "Be careful to not query for attributes that do not exist.\n"
+          ),
+      assi("Question: %<question>s")
+    ].freeze
 
     # The prompt to use for the engine.
     def my_prompt
-      @my_prompt ||= Prompt.new(input_variables: [:question], other_inputs: [:top_k], output_variables: [:answer],
-                                template: TEMPLATE)
+      @conversation ||= Conversation.new(lines: CTEMPLATE)
+      @my_prompt ||= ConversationPrompt.new(
+        conversation: @conversation,
+        input_variables: [:question],
+        other_inputs: [:top_k],
+        output_variables: [:answer])
     end
   end
 end

data/lib/boxcars/boxcar/calculator.rb

@@ -13,19 +13,18 @@ module Boxcars
     def initialize(engine: nil, prompt: nil, **kwargs)
       the_prompt = prompt || my_prompt
       kwargs[:stop] ||= ["```output"]
-      super(name: kwargs[:name] || "Calculator",
-            description: kwargs[:description] || CALCDESC,
-            engine: engine,
-            prompt: the_prompt,
-            **kwargs)
+      kwargs[:name] ||= "Calculator"
+      kwargs[:description] ||= CALCDESC
+      super(engine: engine, prompt: the_prompt, **kwargs)
     end
 
     private
 
     def get_embedded_ruby_answer(text)
-      code = text[8..-4].split("```").first.strip
+      code = text.split("```ruby\n").last.split("```").first.strip
+      # code = text[8..-4].split("```").first.strip
       ruby_executor = Boxcars::RubyREPL.new
-      ruby_executor.call(code: code).strip
+      ruby_executor.call(code: code)
     end
 
     def get_answer(text)
@@ -33,54 +32,42 @@ module Boxcars
       when /^```ruby/
         get_embedded_ruby_answer(text)
       when /^Answer:/
-        text
+        Result.from_text(text)
       else
-        raise Boxcars::Error "Unknown format from engine: #{text}"
+        Result.new(status: :error,
+                   explanation: "Error: expecting your response to begin with '```ruby'. Try answering the question again.")
       end
     end
 
     # our template
-    # rubocop:disable Style/RedundantHeredocDelimiterQuotes
-    TEMPLATE = <<~'IPT'
-      You are GPT-3, and you can't do math.
-      You can do basic math, and your memorization abilities are impressive, but you can't do any complex calculations that a human could not do in their head. You also have an annoying tendency to just make up highly specific, but wrong, answers.
-      So we hooked you up to a Ruby 3 kernel, and now you can execute code written in the Ruby programming language. If anyone gives you a hard math problem, just use this format and we’ll take care of the rest:
-
-      Question: ${{Question with hard calculation.}}
-      ```ruby
-      ${{Code that prints what you need to know}}
-      ```
-      ```output
-      ${{Output of your code}}
-      ```
-      Answer: ${{Answer}}
-
-      Otherwise, use this simpler format:
-
-      Question: ${{Question without hard calculation}}
-      Answer: ${{Answer}}
-
-      Begin.
-
-      Question: What is 37593 * 67?
-      ```ruby
-      puts(37593 * 67)
-      ```
-      ```output
-      2518731
-      ```
-      Answer: 2518731
-
-      Question: what is 2518731 + 0?
-      Answer: 2518731
-
-      Question: %<question>s
-    IPT
-    # rubocop:enable Style/RedundantHeredocDelimiterQuotes
+    CTEMPLATE = [
+      syst("You can do basic math, but for any hard calculations that a human could not do ",
+           "in their head, use the following approach instead. ",
+           "Return code written in the Ruby programming language that prints the results. ",
+           "If anyone gives you a hard math problem, just ",
+           "use the following format and we’ll take care of the rest:\n",
+           "${{Question with hard calculation.}}\n",
+           "reply only with the following format:\n",
+           "```ruby\n${{only Ruby code that prints the answer}}\n```\n",
+           "```output\n${{Output of your code}}\n```\n\n",
+           "Otherwise, you should use this simpler format:\n",
+           "${{Question without hard calculation}}\n",
+           "Answer: ${{Answer}}\n\n",
+           "Do not give an explanation of the answer and make sure your answer starts with either 'Answer:' or '```ruby'."),
+      syst("here is a hard example:\n", "the user asks: What is 37593 * 67?\n",
+           "your answer: ```ruby\nputs(37593 * 67)\n```\n```output\n2518731\n```\nAnswer: 2518731"),
+      syst("basic example:\n", "user asks: What is 2518731 + 0?\n", "you answer: Answer: 2518731"),
+      syst("Begin."),
+      user("%<question>s")
+    ].freeze
 
     # The prompt to use for the engine.
     def my_prompt
-      @my_prompt ||= Prompt.new(input_variables: [:question], output_variables: [:answer], template: TEMPLATE)
+      @conversation ||= Conversation.new(lines: CTEMPLATE)
+      @my_prompt ||= ConversationPrompt.new(
+        conversation: @conversation,
+        input_variables: [:question],
+        output_variables: [:answer])
     end
   end
 end

data/lib/boxcars/boxcar/engine_boxcar.rb

@@ -8,15 +8,14 @@ module Boxcars
 
     # A Boxcar is a container for a single tool to run.
     # @param prompt [Boxcars::Prompt] The prompt to use for this boxcar with sane defaults.
-    # @param name [String] The name of the boxcar. Defaults to classname.
-    # @param description [String] A description of the boxcar.
     # @param engine [Boxcars::Engine] The engine to user for this boxcar. Can be inherited from a train if nil.
-    def initialize(prompt:, engine: nil, name: nil, description: nil, **kwargs)
+    # @param kwargs [Hash] Additional arguments including: name, description, top_k, return_direct, and stop
+    def initialize(prompt:, engine: nil, **kwargs)
       @prompt = prompt
       @engine = engine || Boxcars.engine.new
-      @top_k = kwargs[:top_k] || 5
-      @stop = kwargs[:stop] || ["Answer:"]
-      super(name: name, description: description)
+      @top_k = kwargs.delete(:top_k) || 5
+      @stop = kwargs.delete(:stop) || ["Answer:"]
+      super(**kwargs)
     end
 
     # input keys for the prompt
@@ -34,60 +33,39 @@ module Boxcars
       prompt.output_variables
     end
 
+    # the first output key
+    def output_key
+      output_keys.first
+    end
+
     # generate a response from the engine
     # @param input_list [Array<Hash>] A list of hashes of input values to use for the prompt.
+    # @param current_conversation [Boxcars::Conversation] Optional ongoing conversation to use for the prompt.
     # @return [Boxcars::EngineResult] The result from the engine.
-    def generate(input_list:)
+    def generate(input_list:, current_conversation: nil)
       stop = input_list[0][:stop]
-      prompts = []
-      input_list.each do |inputs|
-        # prompt.missing_variables?(inputs)
-        new_prompt = prompt.format(**inputs)
-        Boxcars.debug("Prompt after formatting:\n#{new_prompt}", :cyan) if Boxcars.configuration.log_prompts
-        prompts.push(new_prompt)
-      end
+      the_prompt = current_conversation ? prompt.with_conversation(current_conversation) : prompt
+      prompts = input_list.map { |inputs| [the_prompt, inputs] }
       engine.generate(prompts: prompts, stop: stop)
     end
 
     # apply a response from the engine
     # @param input_list [Array<Hash>] A list of hashes of input values to use for the prompt.
+    # @param current_conversation [Boxcars::Conversation] Optional ongoing conversation to use for the prompt.
     # @return [Hash] A hash of the output key and the output value.
-    def apply(input_list:)
-      response = generate(input_list: input_list)
+    def apply(input_list:, current_conversation: nil)
+      response = generate(input_list: input_list, current_conversation: current_conversation)
       response.generations.to_h do |generation|
         [output_keys.first, generation[0].text]
       end
     end
 
     # predict a response from the engine
+    # @param current_conversation [Boxcars::Conversation] Optional ongoing conversation to use for the prompt.
     # @param kwargs [Hash] A hash of input values to use for the prompt.
     # @return [String] The output value.
-    def predict(**kwargs)
-      apply(input_list: [kwargs])[output_keys.first]
-    end
-
-    # predict a response from the engine and parse it
-    # @param kwargs [Hash] A hash of input values to use for the prompt.
-    # @return [String] The output value.
-    def predict_and_parse(**kwargs)
-      result = predict(**kwargs)
-      if prompt.output_parser
-        prompt.output_parser.parse(result)
-      else
-        result
-      end
-    end
-
-    # apply a response from the engine and parse it
-    # @param input_list [Array<Hash>] A list of hashes of input values to use for the prompt.
-    # @return [Array<String>] The output values.
-    def apply_and_parse(input_list:)
-      result = apply(input_list: input_list)
-      if prompt.output_parser
-        result.map { |r| prompt.output_parser.parse(r[output_keys.first]) }
-      else
-        result
-      end
+    def predict(current_conversation: nil, **kwargs)
+      apply(current_conversation: current_conversation, input_list: [kwargs])[output_keys.first]
     end
 
     # check that there is exactly one output key
@@ -95,17 +73,35 @@ module Boxcars
     def check_output_keys
       return unless output_keys.length != 1
 
-      raise Boxcars::ArgumentError, "run not supported when there is not exactly one output key. Got #{output_keys}."
+      raise Boxcars::ArgumentError, "not supported when there is not exactly one output key. Got #{output_keys}."
     end
 
     # call the boxcar
     # @param inputs [Hash] The inputs to the boxcar.
     # @return [Hash] The outputs from the boxcar.
     def call(inputs:)
-      t = predict(**prediction_variables(inputs)).strip
-      answer = get_answer(t)
-      Boxcars.debug answer, :magenta
-      { output_keys.first => answer }
+      # if we get errors back, try predicting again giving the errors with the inputs
+      conversation = nil
+      answer = nil
+      4.times do
+        t = predict(current_conversation: conversation, **prediction_variables(inputs)).strip
+        answer = get_answer(t)
+        if answer.status == :error
+          Boxcars.debug "have error, trying again: #{answer.answer}", :red
+          conversation ||= Conversation.new
+          conversation.add_user(answer.answer)
+        else
+          Boxcars.debug answer.to_json, :magenta
+          return { output_keys.first => answer }
+        end
+      end
+      Boxcars.error answer.to_json, :red
+      { output_key => "Error: #{answer}" }
+    rescue Boxcars::ConfigurationError => e
+      raise e
+    rescue Boxcars::Error => e
+      Boxcars.error e.message, :red
+      { output_key => "Error: #{e.message}" }
     end
 
     # @param inputs [Hash] The inputs to the boxcar.