boxcars 0.2.2 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 967f04279864f1419fbe3e9f02817cc8f5594317c6f2b06dd49d1c5e1baa67a9
-  data.tar.gz: 156ca722488b91cefce4d03026490434004e1dcbd1f7c7cd714f1d3f6871cbc0
+  metadata.gz: ea9c5df0584f588d618c22fc44f8bba2638c0946ba32cea8487429ad6df694f5
+  data.tar.gz: a7cd6609da5d63c1b41763fc7c6e8b4e1c8df41992fd133d0bf359b28d50dd0c
 SHA512:
-  metadata.gz: 14158dbd32be6d95e35c2331e26d2e8883bc7621585d5f24b54d1fce6ba78b15e3d67df73b9ed13faf1fd5c306b9341cd4705a3ecf9e5beaf9d47f05e7c1f438
-  data.tar.gz: 90722b9f3b7f34607cd4ee4afe5888c625c5724904943a5160021b48ca1944af5a87269613aca554b595c73c32c923ee9fc442eb8b0271a712fe6aac5702f2ac
+  metadata.gz: aac8bd34e6ddca629d26ff0cb892e46de30e1f20db188b56477422e5526e03c80a22e1b06ab7d8263e8db067db6ec1966f34450ab63e98cea3e857aa7eb27286
+  data.tar.gz: 10ce3be58a020a5de80c27fdf46cbd7811afc45ac28e0712479b113eb6a34ca6a2f7043711659bb4b0a152e0981ded5a8f76fe0704f1a89751b3c8ec0a843ea8

data/CHANGELOG.md

@@ -2,7 +2,15 @@
 
 ## [Unreleased](https://github.com/BoxcarsAI/boxcars/tree/HEAD)
 
-[Full Changelog](https://github.com/BoxcarsAI/boxcars/compare/v0.2.1...HEAD)
+[Full Changelog](https://github.com/BoxcarsAI/boxcars/compare/v0.2.2...HEAD)
+
+**Merged pull requests:**
+
+- better error handling and retry for Active Record Boxcar [\#39](https://github.com/BoxcarsAI/boxcars/pull/39) ([francis](https://github.com/francis))
+
+## [v0.2.2](https://github.com/BoxcarsAI/boxcars/tree/v0.2.2) (2023-03-16)
+
+[Full Changelog](https://github.com/BoxcarsAI/boxcars/compare/v0.2.1...v0.2.2)
 
 **Implemented enhancements:**
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    boxcars (0.2.2)
+    boxcars (0.2.3)
       google_search_results (~> 2.2)
       ruby-openai (~> 3.0)
 

data/lib/boxcars/boxcar/active_record.rb

@@ -71,13 +71,21 @@ module Boxcars
 
     # to be safe, we wrap the code in a transaction and rollback
     def rollback_after_running
-      rv = nil
+      result = nil
+      runtime_exception = nil
       ::ActiveRecord::Base.transaction do
-        rv = yield
+        begin
+          result = yield
+        rescue ::NameError, ::Error => e
+          Boxcars.error("Error while running code: #{e.message[0..60]} ...", :red)
+          runtime_exception = e
+        end
       ensure
         raise ::ActiveRecord::Rollback
       end
-      rv
+      raise runtime_exception if runtime_exception
+
+      result
     end
 
     # check for dangerous code that is outside of ActiveRecord
@@ -122,8 +130,11 @@ module Boxcars
       return true unless changes&.positive?
 
       Boxcars.debug "#{name}(Pending Changes): #{changes}", :yellow
-      change_str = "#{changes} change#{'s' if changes.to_i > 1}"
-      raise SecurityError, "Can not run code that makes #{change_str} in read-only mode" if read_only?
+      if read_only?
+        change_str = "#{changes} change#{'s' if changes.to_i > 1}"
+        Boxcars.error("Can not run code that makes #{change_str} in read-only mode", :red)
+        return false
+      end
 
       return approval_callback.call(changes, code) if approval_callback.is_a?(Proc)
 
@@ -151,30 +162,32 @@ module Boxcars
       output
     end
 
-    def extract_code(code)
-      case code
-      when /^```ruby/
-        code.split('```ruby').last.split('```').first.strip
-      when /^`(.+)`/
-        ::Regexp.last_match(1)
-      else
-        code
-      end
+    def error_message(err, stage)
+      msg = err.message
+      msg = ::Regexp.last_match(1) if msg =~ /^(.+)' for #<Boxcars::ActiveRecord/
+      "#{stage} Error: #{msg} - please fix \"#{stage}:\" to not have this error"
     end
 
     def get_active_record_answer(text)
-      code = extract_code text.split('ARChanges:').first.strip.split('ARCode:').last.strip
-      changes_code = extract_code text.split('ARChanges:').last.strip if text =~ /^ARChanges:/
+      changes_code = extract_code text.split('ARCode:').first.split('ARChanges:').last.strip if text =~ /^ARChanges:/
+      code = extract_code text.split('ARCode:').last.strip
       return Result.new(status: :ok, explanation: "code to run", code: code, changes_code: changes_code) if code_only?
 
-      raise SecurityError, "Permission to run code that makes changes denied" unless approved?(changes_code, code)
+      have_approval = false
+      begin
+        have_approval = approved?(changes_code, code)
+      rescue NameError, ::Error => e
+        return Result.new(status: :error, explanation: error_message(e, "ARChanges"), changes_code: changes_code)
+      end
+
+      raise SecurityError, "Permission to run code that makes changes denied" unless have_approval
 
-      output = clean_up_output(run_active_record_code(code))
-      Result.new(status: :ok, answer: output, explanation: "Answer: #{output.to_json}", code: code)
-    rescue SecurityError, ConfigurationError => e
-      raise e
-    rescue Boxcars::Error => e
-      Result.new(status: :error, answer: nil, explanation: "Error: #{e.message}", code: code)
+      begin
+        output = clean_up_output(run_active_record_code(code))
+        Result.new(status: :ok, answer: output, explanation: "Answer: #{output.to_json}", code: code)
+      rescue NameError, ::Error => e
+        Result.new(status: :error, answer: nil, explanation: error_message(e, "ARCode"), code: code)
+      end
     end
 
     def get_answer(text)
@@ -184,7 +197,8 @@ module Boxcars
       when /^Answer:/
         Result.from_text(text)
       else
-        Result.from_error("Try answering again. Expected your answer to start with 'ARCode:'. You gave me:\n#{text}")
+        Result.from_error("Error: Your answer wasn't formatted properly - try again. I expected your answer to " \
+                          "start with \"ARChanges:\" or \"ARCode:\"")
       end
     end
 
@@ -196,18 +210,19 @@ module Boxcars
            "to at most %<top_k>s results.\n",
            "Never query for all the columns from a specific model, ",
            "only ask for the relevant attributes given the question.\n",
-           "Also, pay attention to which attribute is in which model."),
-      syst("Use the following format:\n",
+           "Also, pay attention to which attribute is in which model.\n\n",
+           "Use the following format:\n",
            "Question: ${{Question here}}\n",
-           "ARCode: ${{Active Record code to run}} - make sure you use valid code\n",
            "ARChanges: ${{Active Record code to compute the number of records going to change}} - ",
-           "Only add this line if the ARCode on the line before will make data changes.\n",
-           "Answer: ${{Final answer here}}"),
-      syst("Only use the following Active Record models: %<model_info>s\n",
-           "Pay attention to use only the attribute names that you can see in the model description. ",
-           "Be careful to not query for attributes that do not exist.\n"
+           "Only add this line if the ARCode on the next line will make data changes.\n",
+           "ARCode: ${{Active Record code to run}} - make sure you use valid code\n",
+           "Answer: ${{Final answer here}}\n\n",
+           "Only use the following Active Record models: %<model_info>s\n",
+           "Pay attention to use only the attribute names that you can see in the model description.\n",
+           "Do not make up variable or attribute names, and do not share variables between the code in ARChanges and ARCode\n",
+           "Be careful to not query for attributes that do not exist, and to use the format specified above.\n"
           ),
-      assi("Question: %<question>s")
+      user("Question: %<question>s")
     ].freeze
 
     # The prompt to use for the engine.

data/lib/boxcars/boxcar/engine_boxcar.rb

@@ -65,7 +65,9 @@ module Boxcars
     # @param kwargs [Hash] A hash of input values to use for the prompt.
     # @return [String] The output value.
     def predict(current_conversation: nil, **kwargs)
-      apply(current_conversation: current_conversation, input_list: [kwargs])[output_keys.first]
+      prediction = apply(current_conversation: current_conversation, input_list: [kwargs])[output_keys.first]
+      Boxcars.debug(prediction, :white) if Boxcars.configuration.log_generated
+      prediction
     end
 
     # check that there is exactly one output key
@@ -84,11 +86,12 @@ module Boxcars
       conversation = nil
       answer = nil
       4.times do
-        t = predict(current_conversation: conversation, **prediction_variables(inputs)).strip
-        answer = get_answer(t)
+        text = predict(current_conversation: conversation, **prediction_variables(inputs)).strip
+        answer = get_answer(text)
         if answer.status == :error
           Boxcars.debug "have error, trying again: #{answer.answer}", :red
           conversation ||= Conversation.new
+          conversation.add_assistant(text)
           conversation.add_user(answer.answer)
         else
           Boxcars.debug answer.to_json, :magenta
@@ -120,5 +123,19 @@ module Boxcars
     def prediction_variables(inputs)
       prediction_input(inputs).merge(prediction_additional)
     end
+
+    # remove backticks or triple backticks from the code
+    # @param code [String] The code to remove backticks from.
+    # @return [String] The code without backticks.
+    def extract_code(code)
+      case code
+      when /^```\w*/
+        code.split(/```\w*\n/).last.split('```').first.strip
+      when /^`(.+)`/
+        ::Regexp.last_match(1)
+      else
+        code.gsub("`", "")
+      end
+    end
   end
 end

data/lib/boxcars/boxcar/sql.rb

@@ -20,6 +20,8 @@ module Boxcars
       kwargs[:name] ||= "Database"
       kwargs[:description] ||= format(SQLDESC, name: name)
       kwargs[:prompt] ||= my_prompt
+      kwargs[:stop] ||= ["SQLResult:"]
+
       super(**kwargs)
     end
 
@@ -73,6 +75,7 @@ module Boxcars
 
     def get_embedded_sql_answer(text)
       code = text[/^SQLQuery: (.*)/, 1]
+      code = extract_code text.split('SQLQuery:').last.strip
       Boxcars.debug code, :yellow
       output = clean_up_output(connection.exec_query(code))
       Result.new(status: :ok, answer: output, explanation: "Answer: #{output.to_json}", code: code)
@@ -87,7 +90,8 @@ module Boxcars
       when /^Answer:/
         Result.from_text(text)
       else
-        Result.from_error("Try answering again. Expected your answer to start with 'SQLQuery:'. You gave me:\n#{text}")
+        Result.from_error("Your answer wasn't formatted properly - try again. I expected your answer to " \
+                          "start with \"SQLQuery:\".")
       end
     end
 
@@ -99,8 +103,8 @@ module Boxcars
            "to return the most interesting examples in the database.\n",
            "Never query for all the columns from a specific table, only ask for the elevant columns given the question.\n",
            "Pay attention to use only the column names that you can see in the schema description. Be careful to ",
-           "not query for columns that do not exist. Also, pay attention to which column is in which table."),
-      syst("Use the following format:\n",
+           "not query for columns that do not exist. Also, pay attention to which column is in which table.\n",
+           "Use the following format:\n",
            "Question: 'Question here'\n",
            "SQLQuery: 'SQL Query to run'\n",
            "SQLResult: 'Result of the SQLQuery'\n",

data/lib/boxcars/train/zero_shot.rb

@@ -49,11 +49,15 @@ module Boxcars
       else
         # the thought should be the frist line here if it doesn't start with "Action:"
         thought = engine_output.split(/\n+/).reject(&:empty?).first
-        Boxcars.debug("Though: #{thought}", :cyan)
-        regex = /Action: (?<action>.*)\nAction Input: (?<action_input>.*)/
+        Boxcars.debug("Thought: #{thought}", :yellow)
+        regex = /Action: (?<action>.*)\n+Action Input: (?<action_input>.*)/
         match = regex.match(engine_output)
-        # TODO: this should return an error to the results that can be used for corrections
-        raise ValueError, "Could not parse engine output: #{engine_output}" unless match
+
+        # we have an unexpected output from the engine
+        unless match
+          return [:error, "You gave me an improperly fomatted answer - try again. For example, if you know the final anwwer, " \
+                          "start with #{FINAL_ANSWER_ACTION.inspect}"]
+        end
 
         action = match[:action].strip
         action_input = match[:action_input].strip.delete_prefix('"').delete_suffix('"')
@@ -63,24 +67,25 @@ module Boxcars
 
     CTEMPLATE = [
       syst("Answer the following questions as best you can. You have access to the following actions:\n",
-           "%<boxcar_descriptions>s"),
-      syst("Use the following format:\n",
+           "%<boxcar_descriptions>s\n",
+           "Use the following format:\n",
            "Question: the input question you must answer\n",
            "Thought: you should always think about what to do\n",
-           "Action: the action to take, should be one of [%<boxcar_names>s]\n",
+           "Action: the action to take, should be one from this list: %<boxcar_names>s\n",
            "Action Input: the input to the action\n",
            "Observation: the result of the action\n",
            "... (this Thought/Action/Action Input/Observation sequence can repeat N times)\n",
-           "Thought: I now know the final answer\n",
+           "Thought: I know the final answer\n",
            "Final Answer: the final answer to the original input question\n",
-           "Next Actions: If you have them, up to 3 suggested actions for the user to take after getting this answer.\n",
+           "Next Actions: Up to 3 logical suggested next questions for the user to ask after getting this answer.\n",
+           "Remember to start a line with \"Final Answer:\" to give me the final answer.\n",
            "Begin!"),
       user("Question: %<input>s"),
       assi("Thought: %<agent_scratchpad>s")
     ].freeze
 
     def boxcar_names
-      @boxcar_names ||= boxcars.map(&:name)
+      @boxcar_names ||= "[#{boxcars.map(&:name).join(', ')}]"
     end
 
     def boxcar_descriptions

data/lib/boxcars/train.rb

@@ -27,6 +27,7 @@ module Boxcars
     # Extract the boxcar name and input from the text.
     # @param text [String] The text to extract from.
     def extract_boxcar_and_input(text)
+      Result.new(status: :ok, answer: text, explanation: engine_output)
     end
 
     # build the scratchpad for the engine
@@ -56,6 +57,7 @@ module Boxcars
       end
       if parsed_output.is_a?(Result)
         TrainAction.from_result(boxcar: "Final Answer", result: parsed_output, log: full_output)
+      # elsif parsed_output[0] == "Error"
       else
         TrainAction.new(boxcar: parsed_output[0], boxcar_input: parsed_output[1], log: full_output)
       end
@@ -198,6 +200,9 @@ module Boxcars
             Boxcars.error "Error in #{boxcar.name} boxcar#call: #{e}", :red
             observation = "Error - #{e}, correct and try again."
           end
+        elsif output.boxcar == :error
+          observation = output.log
+          return_direct = false
         else
           observation = "#{output.boxcar} is not a valid boxcar, try another one."
           return_direct = false

data/lib/boxcars/version.rb

@@ -2,5 +2,5 @@
 
 module Boxcars
   # The current version of the gem.
-  VERSION = "0.2.2"
+  VERSION = "0.2.3"
 end

data/lib/boxcars.rb

@@ -25,12 +25,13 @@ module Boxcars
   # Configuration contains gem settings
   class Configuration
     attr_writer :openai_access_token, :serpapi_api_key
-    attr_accessor :organization_id, :logger, :log_prompts, :default_train, :default_engine
+    attr_accessor :organization_id, :logger, :log_prompts, :log_generated, :default_train, :default_engine
 
     def initialize
       @organization_id = nil
       @logger = Rails.logger if defined?(Rails)
       @log_prompts = ENV.fetch("LOG_PROMPTS", false)
+      @log_generated = ENV.fetch("LOG_GEN", false)
     end
 
     # @return [String] The OpenAI Access Token either from arg or env.

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: boxcars
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.3
 platform: ruby
 authors:
 - Francis Sullivan
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-03-16 00:00:00.000000000 Z
+date: 2023-03-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: debug