boxcars 0.2.1 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 50a47cf449c063e7ca02cb36ad315008fa6a5813063a24d92ed4c0f94743c12e
-  data.tar.gz: fad47ad6fefc7d580251edf1675ae575a9f4eebf28fb903a1c82ba2868310bc4
+  metadata.gz: 967f04279864f1419fbe3e9f02817cc8f5594317c6f2b06dd49d1c5e1baa67a9
+  data.tar.gz: 156ca722488b91cefce4d03026490434004e1dcbd1f7c7cd714f1d3f6871cbc0
 SHA512:
-  metadata.gz: '073919902e84d95158573400ebe593d1e5aa72cc1ab6efb0c4cfccaf9806a91a0c55913d9d9ec47dd8b56f7fb81660fdfd81753092e2a3bd590e4ef782c51ba8'
-  data.tar.gz: 87f1ec995889c7ca107ccf187f1b01874f136787861e5116abc13da5f204f4f3e7e8cf101692fed7285ac875f405e1f8e08701f3785e0c5e83f25d0e5a0b9b15
+  metadata.gz: 14158dbd32be6d95e35c2331e26d2e8883bc7621585d5f24b54d1fce6ba78b15e3d67df73b9ed13faf1fd5c306b9341cd4705a3ecf9e5beaf9d47f05e7c1f438
+  data.tar.gz: 90722b9f3b7f34607cd4ee4afe5888c625c5724904943a5160021b48ca1944af5a87269613aca554b595c73c32c923ee9fc442eb8b0271a712fe6aac5702f2ac

data/CHANGELOG.md

@@ -1,5 +1,32 @@
 # Changelog
 
+## [Unreleased](https://github.com/BoxcarsAI/boxcars/tree/HEAD)
+
+[Full Changelog](https://github.com/BoxcarsAI/boxcars/compare/v0.2.1...HEAD)
+
+**Implemented enhancements:**
+
+- return a structure from boxcars instead of just a string [\#31](https://github.com/BoxcarsAI/boxcars/issues/31)
+- modify SQL boxcar to take a list of Tables. Handy if you have a large database with many tables. [\#19](https://github.com/BoxcarsAI/boxcars/issues/19)
+
+**Closed issues:**
+
+- make a new type of prompt that uses conversations [\#38](https://github.com/BoxcarsAI/boxcars/issues/38)
+- Add test coverage [\#8](https://github.com/BoxcarsAI/boxcars/issues/8)
+
+## [v0.2.1](https://github.com/BoxcarsAI/boxcars/tree/v0.2.1) (2023-03-08)
+
+[Full Changelog](https://github.com/BoxcarsAI/boxcars/compare/v0.2.0...v0.2.1)
+
+**Implemented enhancements:**
+
+- add the ability to use the ChatGPT API - a ChatGPT Boxcar::Engine [\#32](https://github.com/BoxcarsAI/boxcars/issues/32)
+- Prompt simplification [\#29](https://github.com/BoxcarsAI/boxcars/issues/29)
+
+**Merged pull requests:**
+
+- use structured results internally and bring SQL up to parity with ActiveRecord boxcar. [\#36](https://github.com/BoxcarsAI/boxcars/pull/36) ([francis](https://github.com/francis))
+
 ## [v0.2.0](https://github.com/BoxcarsAI/boxcars/tree/v0.2.0) (2023-03-07)
 
 [Full Changelog](https://github.com/BoxcarsAI/boxcars/compare/v0.1.8...v0.2.0)

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    boxcars (0.2.1)
+    boxcars (0.2.2)
       google_search_results (~> 2.2)
       ruby-openai (~> 3.0)
 

data/lib/boxcars/boxcar/active_record.rb

@@ -131,6 +131,7 @@ module Boxcars
     end
 
     def run_active_record_code(code)
+      code = ::Regexp.last_match(1) if code =~ /`(.+)`/
       Boxcars.debug code, :yellow
       if read_only?
         rollback_after_running do
@@ -150,16 +151,29 @@ module Boxcars
       output
     end
 
+    def extract_code(code)
+      case code
+      when /^```ruby/
+        code.split('```ruby').last.split('```').first.strip
+      when /^`(.+)`/
+        ::Regexp.last_match(1)
+      else
+        code
+      end
+    end
+
     def get_active_record_answer(text)
-      code = text[/^ARCode: (.*)/, 1]
-      changes_code = text[/^ARChanges: (.*)/, 1]
+      code = extract_code text.split('ARChanges:').first.strip.split('ARCode:').last.strip
+      changes_code = extract_code text.split('ARChanges:').last.strip if text =~ /^ARChanges:/
       return Result.new(status: :ok, explanation: "code to run", code: code, changes_code: changes_code) if code_only?
 
       raise SecurityError, "Permission to run code that makes changes denied" unless approved?(changes_code, code)
 
       output = clean_up_output(run_active_record_code(code))
       Result.new(status: :ok, answer: output, explanation: "Answer: #{output.to_json}", code: code)
-    rescue StandardError => e
+    rescue SecurityError, ConfigurationError => e
+      raise e
+    rescue Boxcars::Error => e
       Result.new(status: :error, answer: nil, explanation: "Error: #{e.message}", code: code)
     end
 
@@ -170,37 +184,40 @@ module Boxcars
       when /^Answer:/
         Result.from_text(text)
       else
-        Result.from_error("Unknown format from engine: #{text}")
+        Result.from_error("Try answering again. Expected your answer to start with 'ARCode:'. You gave me:\n#{text}")
       end
     end
 
-    TEMPLATE = <<~IPT
-      Given an input question, first create a syntactically correct Rails Active Record code to run,
-      then look at the results of the code and return the answer. Unless the user specifies
-      in her question a specific number of examples she wishes to obtain, limit your code
-      to at most %<top_k>s results.
-
-      Never query for all the columns from a specific model, only ask for a the few relevant attributes given the question.
-
-      Pay attention to use only the attribute names that you can see in the model description. Be careful to not query for attributes that do not exist.
-      Also, pay attention to which attribute is in which model.
-
-      Use the following format:
-      Question: "Question here"
-      ARCode: "Active Record code to run"
-      ARChanges: "Active Record code to compute the number of records going to change" - Only add this line if the ARCode on the line before will make data changes
-      Answer: "Final answer here"
-
-      Only use the following Active Record models:
-      %<model_info>s
-
-      Question: %<question>s
-    IPT
+    CTEMPLATE = [
+      syst("You are a Ruby on Rails Active Record code generator"),
+      syst("Given an input question, first create a syntactically correct Rails Active Record code to run, ",
+           "then look at the results of the code and return the answer. Unless the user specifies ",
+           "in her question a specific number of examples she wishes to obtain, limit your code ",
+           "to at most %<top_k>s results.\n",
+           "Never query for all the columns from a specific model, ",
+           "only ask for the relevant attributes given the question.\n",
+           "Also, pay attention to which attribute is in which model."),
+      syst("Use the following format:\n",
+           "Question: ${{Question here}}\n",
+           "ARCode: ${{Active Record code to run}} - make sure you use valid code\n",
+           "ARChanges: ${{Active Record code to compute the number of records going to change}} - ",
+           "Only add this line if the ARCode on the line before will make data changes.\n",
+           "Answer: ${{Final answer here}}"),
+      syst("Only use the following Active Record models: %<model_info>s\n",
+           "Pay attention to use only the attribute names that you can see in the model description. ",
+           "Be careful to not query for attributes that do not exist.\n"
+          ),
+      assi("Question: %<question>s")
+    ].freeze
 
     # The prompt to use for the engine.
     def my_prompt
-      @my_prompt ||= Prompt.new(input_variables: [:question], other_inputs: [:top_k], output_variables: [:answer],
-                                template: TEMPLATE)
+      @conversation ||= Conversation.new(lines: CTEMPLATE)
+      @my_prompt ||= ConversationPrompt.new(
+        conversation: @conversation,
+        input_variables: [:question],
+        other_inputs: [:top_k],
+        output_variables: [:answer])
     end
   end
 end

data/lib/boxcars/boxcar/calculator.rb

@@ -21,7 +21,8 @@ module Boxcars
     private
 
     def get_embedded_ruby_answer(text)
-      code = text[8..-4].split("```").first.strip
+      code = text.split("```ruby\n").last.split("```").first.strip
+      # code = text[8..-4].split("```").first.strip
       ruby_executor = Boxcars::RubyREPL.new
       ruby_executor.call(code: code)
     end
@@ -33,52 +34,40 @@ module Boxcars
       when /^Answer:/
         Result.from_text(text)
       else
-        Result.new(status: :error, explanation: "Unknown format from engine: #{text}")
+        Result.new(status: :error,
+                   explanation: "Error: expecting your response to begin with '```ruby'. Try answering the question again.")
       end
     end
 
     # our template
-    # rubocop:disable Style/RedundantHeredocDelimiterQuotes
-    TEMPLATE = <<~'IPT'
-      You are GPT-3, and you can't do math.
-      You can do basic math, and your memorization abilities are impressive, but you can't do any complex calculations that a human could not do in their head. You also have an annoying tendency to just make up highly specific, but wrong, answers.
-      So we hooked you up to a Ruby 3 kernel, and now you can execute code written in the Ruby programming language. If anyone gives you a hard math problem, just use this format and we’ll take care of the rest:
-
-      Question: ${{Question with hard calculation.}}
-      ```ruby
-      ${{Code that prints what you need to know}}
-      ```
-      ```output
-      ${{Output of your code}}
-      ```
-      Answer: ${{Answer}}
-
-      Otherwise, use this simpler format:
-
-      Question: ${{Question without hard calculation}}
-      Answer: ${{Answer}}
-
-      Begin.
-
-      Question: What is 37593 * 67?
-      ```ruby
-      puts(37593 * 67)
-      ```
-      ```output
-      2518731
-      ```
-      Answer: 2518731
-
-      Question: what is 2518731 + 0?
-      Answer: 2518731
-
-      Question: %<question>s
-    IPT
-    # rubocop:enable Style/RedundantHeredocDelimiterQuotes
+    CTEMPLATE = [
+      syst("You can do basic math, but for any hard calculations that a human could not do ",
+           "in their head, use the following approach instead. ",
+           "Return code written in the Ruby programming language that prints the results. ",
+           "If anyone gives you a hard math problem, just ",
+           "use the following format and we’ll take care of the rest:\n",
+           "${{Question with hard calculation.}}\n",
+           "reply only with the following format:\n",
+           "```ruby\n${{only Ruby code that prints the answer}}\n```\n",
+           "```output\n${{Output of your code}}\n```\n\n",
+           "Otherwise, you should use this simpler format:\n",
+           "${{Question without hard calculation}}\n",
+           "Answer: ${{Answer}}\n\n",
+           "Do not give an explanation of the answer and make sure your answer starts with either 'Answer:' or '```ruby'."),
+      syst("here is a hard example:\n", "the user asks: What is 37593 * 67?\n",
+           "your answer: ```ruby\nputs(37593 * 67)\n```\n```output\n2518731\n```\nAnswer: 2518731"),
+      syst("basic example:\n", "user asks: What is 2518731 + 0?\n", "you answer: Answer: 2518731"),
+      syst("Begin."),
+      user("%<question>s")
+    ].freeze
 
     # The prompt to use for the engine.
     def my_prompt
-      @my_prompt ||= Prompt.new(input_variables: [:question], output_variables: [:answer], template: TEMPLATE)
+      @conversation ||= Conversation.new(lines: CTEMPLATE)
+      @my_prompt ||= ConversationPrompt.new(
+        conversation: @conversation,
+        input_variables: [:question],
+        output_variables: [:answer])
     end
   end
 end

data/lib/boxcars/boxcar/engine_boxcar.rb

@@ -8,15 +8,14 @@ module Boxcars
 
     # A Boxcar is a container for a single tool to run.
     # @param prompt [Boxcars::Prompt] The prompt to use for this boxcar with sane defaults.
-    # @param name [String] The name of the boxcar. Defaults to classname.
-    # @param description [String] A description of the boxcar.
     # @param engine [Boxcars::Engine] The engine to user for this boxcar. Can be inherited from a train if nil.
-    def initialize(prompt:, engine: nil, name: nil, description: nil, **kwargs)
+    # @param kwargs [Hash] Additional arguments including: name, description, top_k, return_direct, and stop
+    def initialize(prompt:, engine: nil, **kwargs)
       @prompt = prompt
       @engine = engine || Boxcars.engine.new
-      @top_k = kwargs[:top_k] || 5
-      @stop = kwargs[:stop] || ["Answer:"]
-      super(name: name, description: description, return_direct: kwargs[:return_direct])
+      @top_k = kwargs.delete(:top_k) || 5
+      @stop = kwargs.delete(:stop) || ["Answer:"]
+      super(**kwargs)
     end
 
     # input keys for the prompt
@@ -34,60 +33,39 @@ module Boxcars
       prompt.output_variables
     end
 
+    # the first output key
+    def output_key
+      output_keys.first
+    end
+
     # generate a response from the engine
     # @param input_list [Array<Hash>] A list of hashes of input values to use for the prompt.
+    # @param current_conversation [Boxcars::Conversation] Optional ongoing conversation to use for the prompt.
     # @return [Boxcars::EngineResult] The result from the engine.
-    def generate(input_list:)
+    def generate(input_list:, current_conversation: nil)
       stop = input_list[0][:stop]
-      prompts = []
-      input_list.each do |inputs|
-        # prompt.missing_variables?(inputs)
-        new_prompt = prompt.format(**inputs)
-        Boxcars.debug("Prompt after formatting:\n#{new_prompt}", :cyan) if Boxcars.configuration.log_prompts
-        prompts.push(new_prompt)
-      end
+      the_prompt = current_conversation ? prompt.with_conversation(current_conversation) : prompt
+      prompts = input_list.map { |inputs| [the_prompt, inputs] }
       engine.generate(prompts: prompts, stop: stop)
     end
 
     # apply a response from the engine
     # @param input_list [Array<Hash>] A list of hashes of input values to use for the prompt.
+    # @param current_conversation [Boxcars::Conversation] Optional ongoing conversation to use for the prompt.
     # @return [Hash] A hash of the output key and the output value.
-    def apply(input_list:)
-      response = generate(input_list: input_list)
+    def apply(input_list:, current_conversation: nil)
+      response = generate(input_list: input_list, current_conversation: current_conversation)
       response.generations.to_h do |generation|
         [output_keys.first, generation[0].text]
       end
     end
 
     # predict a response from the engine
+    # @param current_conversation [Boxcars::Conversation] Optional ongoing conversation to use for the prompt.
     # @param kwargs [Hash] A hash of input values to use for the prompt.
     # @return [String] The output value.
-    def predict(**kwargs)
-      apply(input_list: [kwargs])[output_keys.first]
-    end
-
-    # predict a response from the engine and parse it
-    # @param kwargs [Hash] A hash of input values to use for the prompt.
-    # @return [String] The output value.
-    def predict_and_parse(**kwargs)
-      result = predict(**kwargs)
-      if prompt.output_parser
-        prompt.output_parser.parse(result)
-      else
-        result
-      end
-    end
-
-    # apply a response from the engine and parse it
-    # @param input_list [Array<Hash>] A list of hashes of input values to use for the prompt.
-    # @return [Array<String>] The output values.
-    def apply_and_parse(input_list:)
-      result = apply(input_list: input_list)
-      if prompt.output_parser
-        result.map { |r| prompt.output_parser.parse(r[output_keys.first]) }
-      else
-        result
-      end
+    def predict(current_conversation: nil, **kwargs)
+      apply(current_conversation: current_conversation, input_list: [kwargs])[output_keys.first]
     end
 
     # check that there is exactly one output key
@@ -102,10 +80,28 @@ module Boxcars
     # @param inputs [Hash] The inputs to the boxcar.
     # @return [Hash] The outputs from the boxcar.
     def call(inputs:)
-      t = predict(**prediction_variables(inputs)).strip
-      answer = get_answer(t)
-      Boxcars.debug answer.to_json, :magenta
-      { output_keys.first => answer }
+      # if we get errors back, try predicting again giving the errors with the inputs
+      conversation = nil
+      answer = nil
+      4.times do
+        t = predict(current_conversation: conversation, **prediction_variables(inputs)).strip
+        answer = get_answer(t)
+        if answer.status == :error
+          Boxcars.debug "have error, trying again: #{answer.answer}", :red
+          conversation ||= Conversation.new
+          conversation.add_user(answer.answer)
+        else
+          Boxcars.debug answer.to_json, :magenta
+          return { output_keys.first => answer }
+        end
+      end
+      Boxcars.error answer.to_json, :red
+      { output_key => "Error: #{answer}" }
+    rescue Boxcars::ConfigurationError => e
+      raise e
+    rescue Boxcars::Error => e
+      Boxcars.error e.message, :red
+      { output_key => "Error: #{e.message}" }
     end
 
     # @param inputs [Hash] The inputs to the boxcar.

data/lib/boxcars/boxcar/sql.rb

@@ -87,41 +87,36 @@ module Boxcars
       when /^Answer:/
         Result.from_text(text)
       else
-        Result.from_error("Unknown format from engine: #{text}")
+        Result.from_error("Try answering again. Expected your answer to start with 'SQLQuery:'. You gave me:\n#{text}")
       end
     end
 
-    TEMPLATE = <<~IPT
-      Given an input question, first create a syntactically correct %<dialect>s SQL query to run,
-      then look at the results of the query and return the answer. Unless the user specifies
-      in his question a specific number of examples he wishes to obtain, always limit your query
-      to at most %<top_k>s results using a LIMIT clause. You can order the results by a relevant column
-      to return the most interesting examples in the database.
-
-      Never query for all the columns from a specific table, only ask for a the few relevant columns given the question.
-
-      Pay attention to use only the column names that you can see in the schema description. Be careful to not query for columns that do not exist.
-      Also, pay attention to which column is in which table.
-
-      Use the following format:
-      Question: "Question here"
-      SQLQuery: "SQL Query to run"
-      SQLResult: "Result of the SQLQuery"
-      Answer: "Final answer here"
-
-      Only use the following tables:
-      %<schema>s
-
-      Question: %<question>s
-    IPT
+    CTEMPLATE = [
+      syst("Given an input question, first create a syntactically correct %<dialect>s SQL query to run, ",
+           "then look at the results of the query and return the answer. Unless the user specifies ",
+           "in her question a specific number of examples he wishes to obtain, always limit your query ",
+           "to at most %<top_k>s results using a LIMIT clause. You can order the results by a relevant column ",
+           "to return the most interesting examples in the database.\n",
+           "Never query for all the columns from a specific table, only ask for the elevant columns given the question.\n",
+           "Pay attention to use only the column names that you can see in the schema description. Be careful to ",
+           "not query for columns that do not exist. Also, pay attention to which column is in which table."),
+      syst("Use the following format:\n",
+           "Question: 'Question here'\n",
+           "SQLQuery: 'SQL Query to run'\n",
+           "SQLResult: 'Result of the SQLQuery'\n",
+           "Answer: 'Final answer here'"),
+      syst("Only use the following tables:\n%<schema>s"),
+      user("Question: %<question>s")
+    ].freeze
 
     # The prompt to use for the engine.
     def my_prompt
-      @my_prompt ||= Prompt.new(
+      @conversation ||= Conversation.new(lines: CTEMPLATE)
+      @my_prompt ||= ConversationPrompt.new(
+        conversation: @conversation,
         input_variables: [:question],
         other_inputs: [:top_k, :dialect, :table_info],
-        output_variables: [:answer],
-        template: TEMPLATE)
+        output_variables: [:answer])
     end
   end
 end

data/lib/boxcars/boxcar.rb

@@ -78,6 +78,22 @@ module Boxcars
       rv
     end
 
+    # helpers for conversation prompt building
+    # assistant message
+    def self.assi(*strs)
+      [:assistant, strs.join]
+    end
+
+    # system message
+    def self.syst(*strs)
+      [:system, strs.join]
+    end
+
+    # user message
+    def self.user(*strs)
+      [:user, strs.join]
+    end
+
     private
 
     # Get an answer from the boxcar.

data/lib/boxcars/conversation.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module Boxcars
+  # used to keep track of the conversation
+  class Conversation
+    attr_reader :lines, :show_roles
+
+    PEOPLE = %i[system user assistant].freeze
+
+    def initialize(lines: [], show_roles: false)
+      @lines = lines
+      check_lines(@lines)
+      @show_roles = show_roles
+    end
+
+    # check the lines
+    def check_lines(lines)
+      raise ArgumentError, "Lines must be an array" unless lines.is_a?(Array)
+
+      lines.each do |ln|
+        raise ArgumentError, "Conversation item must be a array" unless ln.is_a?(Array)
+        raise ArgumentError, "Conversation item must have 2 items, role and text" unless ln.size == 2
+        raise ArgumentError, "Conversation item must have a role #{ln} in (#{PEOPLE})" unless PEOPLE.include? ln[0]
+        raise ArgumentError, "Conversation value must be a string" unless ln[1].is_a?(String)
+      end
+    end
+
+    # @return [Array] The result as a convesation array
+    def to_a
+      lines
+    end
+
+    # @return [String] A conversation string
+    def to_s
+      lines.map { |ln| "#{ln[0]}: #{ln[1]}" }.join("\n")
+    end
+
+    # add assistant text to the conversation at the end
+    # @param text [String] The text to add
+    def add_assistant(text)
+      @lines << [:assistant, text]
+    end
+
+    # add user text to the conversation at the end
+    # @param text [String] The text to add
+    def add_user(text)
+      @lines << [:user, text]
+    end
+
+    # add system text to the conversation at the end
+    # @param text [String] The text to add
+    def add_system(text)
+      @lines << [:system, text]
+    end
+
+    # add multiple lines to the conversation
+    def add_lines(lines)
+      check_lines(lines)
+      @lines += lines
+    end
+
+    # add a conversation to the conversation
+    def add_conversation(conversation)
+      @lines += conversation.lines
+    end
+
+    # return just the messages for the conversation
+    def message_text
+      lines.map(&:last).join("\n")
+    end
+
+    # compute the prompt parameters with input substitutions (used for chatGPT)
+    # @param inputs [Hash] The inputs to use for the prompt.
+    # @return [Hash] The formatted prompt { messages: ...}
+    def as_messages(inputs = nil)
+      { messages: lines.map { |ln| { role: ln.first, content: ln.last % inputs } } }
+    rescue ::KeyError => e
+      first_line = e.message.to_s.split("\n").first
+      Boxcars.error "Missing prompt input key: #{first_line}"
+      raise KeyError, "Prompt format error: #{first_line}"
+    end
+
+    # compute the prompt parameters with input substitutions
+    # @param inputs [Hash] The inputs to use for the prompt.
+    # @return [Hash] The formatted prompt { prompt: "..."}
+    def as_prompt(inputs = nil)
+      if show_roles
+        lines.map { |ln| format("#{ln.first}: #{ln.last}", inputs) }.join("\n\n")
+      else
+        lines.map { |ln| format(ln.last, inputs) }.join("\n\n")
+      end
+    rescue ::KeyError => e
+      first_line = e.message.to_s.split("\n").first
+      Boxcars.error "Missing prompt input key: #{first_line}"
+      raise KeyError, "Prompt format error: #{first_line}"
+    end
+  end
+end

data/lib/boxcars/conversation_prompt.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Boxcars
+  # used by Boxcars that have engine's to create a conversation prompt.
+  class ConversationPrompt < Prompt
+    attr_reader :conversation
+
+    # @param conversation [Boxcars::Conversation] The conversation to use for the prompt.
+    # @param input_variables [Array<Symbol>] The input vars to use for the prompt. Defaults to [:input]
+    # @param other_inputs [Array<Symbol>] The other input vars to use for the prompt. Defaults to []
+    # @param output_variables [Array<Symbol>] The output vars to use for the prompt. Defaults to [:output]
+    def initialize(conversation:, input_variables: nil, other_inputs: nil, output_variables: nil)
+      @conversation = conversation
+      super(template: template, input_variables: input_variables, other_inputs: other_inputs, output_variables: output_variables)
+    end
+
+    # prompt for chatGPT params
+    # @param inputs [Hash] The inputs to use for the prompt.
+    # @return [Hash] The formatted prompt.
+    def as_messages(inputs)
+      conversation.as_messages(inputs)
+    end
+
+    # prompt for non chatGPT params
+    # @param inputs [Hash] The inputs to use for the prompt.
+    # @return [Hash] The formatted prompt.
+    def as_prompt(inputs)
+      { prompt: conversation.as_prompt(inputs) }
+    end
+
+    # tack on the ongoing conversation if present to the prompt
+    def with_conversation(conversation)
+      return self unless conversation
+
+      new_prompt = dup
+      new_prompt.conversation.add_conversation(conversation)
+      new_prompt
+    end
+  end
+end

data/lib/boxcars/engine/openai.rb

@@ -10,7 +10,7 @@ module Boxcars
     # The default parameters to use when asking the engine.
     DEFAULT_PARAMS = {
       model: "gpt-3.5-turbo",
-      temperature: 0.7,
+      temperature: 0.2,
       max_tokens: 512
     }.freeze
 
@@ -38,17 +38,22 @@ module Boxcars
     # @param openai_access_token [String] The access token to use when asking the engine.
     #   Defaults to Boxcars.configuration.openai_access_token.
     # @param kwargs [Hash] Additional parameters to pass to the engine if wanted.
-    def client(prompt:, openai_access_token: nil, **kwargs)
+    def client(prompt:, inputs: {}, openai_access_token: nil, **kwargs)
       access_token = Boxcars.configuration.openai_access_token(openai_access_token: openai_access_token)
       organization_id = Boxcars.configuration.organization_id
       clnt = ::OpenAI::Client.new(access_token: access_token, organization_id: organization_id)
-      the_params = { prompt: prompt }.merge(open_ai_params).merge(kwargs)
-      if the_params[:model] == "gpt-3.5-turbo"
+      params = open_ai_params.merge(kwargs)
+      if params[:model] == "gpt-3.5-turbo"
         prompt = prompt.first if prompt.is_a?(Array)
-        the_params = { messages: [{ role: "user", content: prompt }] }.merge(open_ai_params).merge(kwargs)
-        clnt.chat(parameters: the_params)
+        params = prompt.as_messages(inputs).merge(params)
+        if Boxcars.configuration.log_prompts
+          Boxcars.debug(params[:messages].map { |p| ">>>>>> Role: #{p[:role]} <<<<<<\n#{p[:content]}" }.join("\n"), :cyan)
+        end
+        clnt.chat(parameters: params)
       else
-        clnt.completions(parameters: the_params)
+        params = prompt.as_prompt(inputs).merge(params)
+        Boxcars.debug("Prompt after formatting:\n#{params[:prompt]}", :cyan) if Boxcars.configuration.log_prompts
+        clnt.completions(parameters: params)
       end
     end
 
@@ -56,7 +61,8 @@ module Boxcars
     # @param question [String] The question to ask the engine.
     # @param kwargs [Hash] Additional parameters to pass to the engine if wanted.
     def run(question, **kwargs)
-      response = client(prompt: question, **kwargs)
+      prompt = Prompt.new(template: question)
+      response = client(prompt: prompt, **kwargs)
       answer = response["choices"].map { |c| c.dig("message", "content") || c["text"] }.join("\n").strip
       puts answer
       answer
@@ -110,6 +116,7 @@ module Boxcars
 
     # Call out to OpenAI's endpoint with k unique prompts.
     # @param prompts [Array<String>] The prompts to pass into the model.
+    # @param inputs [Array<String>] The inputs to subsitite into the prompt.
     # @param stop [Array<String>] Optional list of stop words to use when generating.
     # @return [EngineResult] The full engine output.
     def generate(prompts:, stop: nil)
@@ -120,13 +127,14 @@ module Boxcars
       # Get the token usage from the response.
       # Includes prompt, completion, and total tokens used.
       inkeys = %w[completion_tokens prompt_tokens total_tokens].freeze
-      sub_prompts = prompts.each_slice(batch_size).to_a
-      sub_prompts.each do |sprompts|
-        response = client(prompt: sprompts, **params)
-        check_response(response)
-        choices.concat(response["choices"])
-        keys_to_use = inkeys & response["usage"].keys
-        keys_to_use.each { |key| token_usage[key] = token_usage[key].to_i + response["usage"][key] }
+      prompts.each_slice(batch_size) do |sub_prompts|
+        sub_prompts.each do |sprompts, inputs|
+          response = client(prompt: sprompts, inputs: inputs, **params)
+          check_response(response)
+          choices.concat(response["choices"])
+          usage_keys = inkeys & response["usage"].keys
+          usage_keys.each { |key| token_usage[key] = token_usage[key].to_i + response["usage"][key] }
+        end
       end
 
       n = params.fetch(:n, 1)

data/lib/boxcars/prompt.rb

@@ -16,6 +16,31 @@ module Boxcars
       @output_variables = output_variables || [:output]
     end
 
+    # compute the prompt parameters with input substitutions (used for chatGPT)
+    # @param inputs [Hash] The inputs to use for the prompt.
+    # @return [Hash] The formatted prompt { messages: ...}
+    def as_prompt(inputs)
+      { prompt: format(inputs) }
+    end
+
+    # compute the prompt parameters with input substitutions
+    # @param inputs [Hash] The inputs to use for the prompt.
+    # @return [Hash] The formatted prompt { prompt: "..."}
+    def as_messages(inputs)
+      { messages: [{ role: :assistant, content: format(inputs) }] }
+    end
+
+    # tack on the ongoing conversation if present to the prompt
+    def with_conversation(conversation)
+      return self unless conversation
+
+      new_prompt = dup
+      new_prompt.template += "\n\n#{conversation.message_text}"
+      new_prompt
+    end
+
+    private
+
     # format the prompt with the input variables
     # @param inputs [Hash] The inputs to use for the prompt.
     # @return [String] The formatted prompt.
@@ -27,43 +52,5 @@ module Boxcars
       Boxcars.error "Missing prompt input key: #{first_line}"
       raise KeyError, "Prompt format error: #{first_line}"
     end
-
-    # check if the template is valid
-    def template_is_valid?
-      all_vars = (input_variables + other_inputs + output_variables).sort
-      template_vars = @template.scan(/%<(\w+)>s/).flatten.map(&:to_sym).sort
-      all_vars == template_vars
-    end
-
-    # missing variables in the template
-    def missing_variables?(inputs)
-      input_vars = [input_variables, other_inputs].flatten.sort
-      return if inputs.keys.sort == input_vars
-
-      raise ArgumentError, "Missing expected input keys, got: #{inputs.keys}. Expected: #{input_vars}"
-    end
-
-    # create a prompt template from examples
-    # @param examples [String] or [Array<String>] The example(s) to use for the template.
-    # @param input_variables [Array<Symbol>] The input variables to use for the prompt.
-    # @param example_separator [String] The separator to use between the examples. Defaults to "\n\n"
-    # @param prefix [String] The prefix to use for the template. Defaults to ""
-    def self.from_examples(examples:, suffix:, input_variables:, example_separator: "\n\n", prefix: "", **kwargs)
-      template = [prefix, examples, suffix].join(example_separator)
-      other_inputs = kwargs[:other_inputs] || []
-      output_variables = kwargs[:output_variables] || [:output]
-      Prompt.new(template: template, input_variables: input_variables, other_inputs: other_inputs,
-                 output_variables: output_variables)
-    end
-
-    # create a prompt template from a file
-    # @param path [String] The path to the file to use for the template.
-    # @param input_variables [Array<Symbol>] The input variables to use for the prompt. Defaults to [:input]
-    # @param output_variables [Array<Symbol>] The output variables to use for the prompt. Defaults to [:output]
-    def self.from_file(path:, input_variables: nil, other_inputs: nil, output_variables: nil)
-      template = File.read(path)
-      Prompt.new(template: template, input_variables: input_variables, other_inputs: other_inputs,
-                 output_variables: output_variables)
-    end
   end
 end

data/lib/boxcars/train/zero_shot.rb

@@ -1,35 +1,11 @@
+# frozen_string_literal: true
+
 # Agent for the MRKL chain
 module Boxcars
   # A Train using the zero-shot react method.
   class ZeroShot < Train
     attr_reader :boxcars, :observation_prefix, :engine_prefix
 
-    # default prompt prefix
-    PREFIX = "Answer the following questions as best you can. You have access to the following actions:".freeze
-
-    # default prompt instructions
-    FORMAT_INSTRUCTIONS = <<~FINPUT.freeze
-      Use the following format:
-
-      Question: the input question you must answer
-      Thought: you should always think about what to do
-      Action: the action to take, should be one of [%<boxcar_names>s]
-      Action Input: the input to the action
-      Observation: the result of the action
-      ... (this Thought/Action/Action Input/Observation sequence can repeat N times)
-      Thought: I now know the final answer
-      Final Answer: the final answer to the original input question
-      Next Actions: If you have them, up to three suggested actions for the user to take after getting this answer.
-    FINPUT
-
-    # default prompt suffix
-    SUFFIX = <<~SINPUT.freeze
-      Begin!
-
-      Question: %<input>s
-      Thought:%<agent_scratchpad>s
-    SINPUT
-
     # @param boxcars [Array<Boxcars::Boxcar>] The boxcars to run.
     # @param engine [Boxcars::Engine] The engine to use for this train.
     # @param name [String] The name of the train. Defaults to 'Zero Shot'.
@@ -38,26 +14,26 @@ module Boxcars
     def initialize(boxcars:, engine: nil, name: 'Zero Shot', description: 'Zero Shot Train', prompt: nil)
       @observation_prefix = 'Observation: '
       @engine_prefix = 'Thought:'
-      prompt ||= self.class.create_prompt(boxcars: boxcars)
+      prompt ||= my_prompt
       super(engine: engine, boxcars: boxcars, prompt: prompt, name: name, description: description)
     end
 
-    # Create prompt in the style of the zero shot agent. Without arguments, returns the default prompt.
-    # @param boxcars [Array<Boxcars::Boxcar>] List of boxcars the agent will have access to, used to format the prompt.
-    # @param prefix [String] String to put before the main prompt.
-    # @param suffix [String] String to put after the main prompt.
-    # @param input_variables [Array<Symbol>] List of input variables the final prompt will expect.
-    # @return [Boxcars::Prompt] A Prompt with the template assembled from the pieces here.
-    def self.create_prompt(boxcars:, prefix: PREFIX, suffix: SUFFIX, input_variables: [:input, :agent_scratchpad])
-      boxcar_strings = boxcars.map { |boxcar| "#{boxcar.name}: #{boxcar.description}" }.join("\n")
-      boxcar_names = boxcars.map(&:name)
-      format_instructions = format(FORMAT_INSTRUCTIONS, boxcar_names: boxcar_names.join(", "))
-      template = [prefix, boxcar_strings, format_instructions, suffix].join("\n\n")
-      Prompt.new(template: template, input_variables: input_variables)
+    # @return Hash The additional variables for this boxcar.
+    def prediction_additional
+      { boxcar_names: boxcar_names, boxcar_descriptions: boxcar_descriptions }.merge super
+    end
+
+    # Extract the boxcar and input from the engine output.
+    # @param text [String] The output from the engine.
+    # @return [Array<Boxcars::Boxcar, String>] The boxcar and input.
+    def extract_boxcar_and_input(text)
+      get_action_and_input(engine_output: text)
     end
 
+    private
+
     # the final answer action string
-    FINAL_ANSWER_ACTION = "Final Answer:".freeze
+    FINAL_ANSWER_ACTION = "Final Answer:"
 
     # Parse out the action and input from the engine output.
     # @param engine_output [String] The output from the engine.
@@ -70,13 +46,13 @@ module Boxcars
       if engine_output.include?(FINAL_ANSWER_ACTION)
         answer = engine_output.split(FINAL_ANSWER_ACTION).last.strip
         Result.new(status: :ok, answer: answer, explanation: engine_output)
-        # ['Final Answer', answer]
       else
         # the thought should be the frist line here if it doesn't start with "Action:"
         thought = engine_output.split(/\n+/).reject(&:empty?).first
         Boxcars.debug("Though: #{thought}", :cyan)
         regex = /Action: (?<action>.*)\nAction Input: (?<action_input>.*)/
         match = regex.match(engine_output)
+        # TODO: this should return an error to the results that can be used for corrections
         raise ValueError, "Could not parse engine output: #{engine_output}" unless match
 
         action = match[:action].strip
@@ -85,11 +61,40 @@ module Boxcars
       end
     end
 
-    # Extract the boxcar and input from the engine output.
-    # @param text [String] The output from the engine.
-    # @return [Array<Boxcars::Boxcar, String>] The boxcar and input.
-    def extract_boxcar_and_input(text)
-      get_action_and_input(engine_output: text)
+    CTEMPLATE = [
+      syst("Answer the following questions as best you can. You have access to the following actions:\n",
+           "%<boxcar_descriptions>s"),
+      syst("Use the following format:\n",
+           "Question: the input question you must answer\n",
+           "Thought: you should always think about what to do\n",
+           "Action: the action to take, should be one of [%<boxcar_names>s]\n",
+           "Action Input: the input to the action\n",
+           "Observation: the result of the action\n",
+           "... (this Thought/Action/Action Input/Observation sequence can repeat N times)\n",
+           "Thought: I now know the final answer\n",
+           "Final Answer: the final answer to the original input question\n",
+           "Next Actions: If you have them, up to 3 suggested actions for the user to take after getting this answer.\n",
+           "Begin!"),
+      user("Question: %<input>s"),
+      assi("Thought: %<agent_scratchpad>s")
+    ].freeze
+
+    def boxcar_names
+      @boxcar_names ||= boxcars.map(&:name)
+    end
+
+    def boxcar_descriptions
+      @boxcar_descriptions ||= boxcars.map { |boxcar| "#{boxcar.name}: #{boxcar.description}" }.join("\n")
+    end
+
+    # The prompt to use for the train.
+    def my_prompt
+      @conversation ||= Conversation.new(lines: CTEMPLATE)
+      @my_prompt ||= ConversationPrompt.new(
+        conversation: @conversation,
+        input_variables: [:input],
+        other_inputs: [:boxcar_names, :boxcar_descriptions, :agent_scratchpad],
+        output_variables: [:answer])
     end
   end
 end

data/lib/boxcars/train.rb

@@ -3,23 +3,25 @@
 module Boxcars
   # @abstract
   class Train < EngineBoxcar
-    attr_reader :engine, :boxcars, :name, :description, :prompt, :return_values, :return_intermediate_steps,
-                :max_iterations, :early_stopping_method
+    attr_reader :boxcars, :return_values, :return_intermediate_steps,
+                :max_iterations, :early_stopping_method, :name_to_boxcar_map
 
     # A Train will use a engine to run a series of boxcars.
-    # @param engine [Boxcars::Engine] The engine to use for this train.
     # @param boxcars [Array<Boxcars::Boxcar>] The boxcars to run.
-    # @param prompt [String] The prompt to use.
+    # @param prompt [Boxcars::Prompt] The prompt to use.
+    # @param engine [Boxcars::Engine] The engine to use for this train.
+    # @param kwargs [Hash] Additional arguments including: name, description, top_k, return_direct, and stop
     # @abstract
     def initialize(boxcars:, prompt:, engine: nil, **kwargs)
       @boxcars = boxcars
-      @name = name || self.class.name
+      @name_to_boxcar_map = boxcars.to_h { |boxcar| [boxcar.name, boxcar] }
       @return_values = [:output]
-      @return_intermediate_steps = kwargs[:return_intermediate_steps] || false
-      @max_iterations = kwargs[:max_iterations] || 25
-      @early_stopping_method = kwargs[:early_stopping_method] || "force"
+      @return_intermediate_steps = kwargs.delete(:return_intermediate_steps) || false
+      @max_iterations = kwargs.delete(:max_iterations) || 25
+      @early_stopping_method = kwargs.delete(:early_stopping_method) || "force"
+      kwargs[:stop] ||= ["\n#{observation_prefix}"]
 
-      super(prompt: prompt, engine: engine, name: kwargs[:name], description: kwargs[:description])
+      super(prompt: prompt, engine: engine, **kwargs)
     end
 
     # Extract the boxcar name and input from the text.
@@ -27,11 +29,6 @@ module Boxcars
     def extract_boxcar_and_input(text)
     end
 
-    # the stop strings list
-    def stop
-      ["\n#{observation_prefix}"]
-    end
-
     # build the scratchpad for the engine
     # @param intermediate_steps [Array] The intermediate steps to build the scratchpad from.
     # @return [String] The scratchpad.
@@ -48,13 +45,14 @@ module Boxcars
     # @param full_inputs [Hash] The inputs to the engine.
     # @return [Boxcars::Action] The next action.
     def get_next_action(full_inputs)
-      full_output = predict(**full_inputs)
-      parsed_output = extract_boxcar_and_input(full_output)
-      while parsed_output.nil?
+      full_output = ""
+      parsed_output = nil
+      loop do
         full_inputs[:agent_scratchpad] += full_output
         output = predict(**full_inputs)
         full_output += output
         parsed_output = extract_boxcar_and_input(full_output)
+        break unless parsed_output.nil?
       end
       if parsed_output.is_a?(Result)
         TrainAction.from_result(boxcar: "Final Answer", result: parsed_output, log: full_output)
@@ -69,8 +67,7 @@ module Boxcars
     # @return [Boxcars::Action] Action specifying what boxcar to use.
     def plan(intermediate_steps, **kwargs)
       thoughts = construct_scratchpad(intermediate_steps)
-      new_inputs = { agent_scratchpad: thoughts, stop: stop }
-      full_inputs = kwargs.merge(new_inputs)
+      full_inputs = prediction_additional.merge(kwargs).merge(agent_scratchpad: thoughts)
       action = get_next_action(full_inputs)
       return TrainFinish.new({ output: action.boxcar_input }, log: action.log) if action.boxcar == finish_boxcar_name
 
@@ -187,7 +184,6 @@ module Boxcars
     # @return [Hash] The output.
     def call(inputs:)
       prepare_for_new_call
-      name_to_boxcar_map = boxcars.to_h { |boxcar| [boxcar.name, boxcar] }
       intermediate_steps = []
       iterations = 0
       while should_continue?(iterations)

data/lib/boxcars/version.rb

@@ -2,5 +2,5 @@
 
 module Boxcars
   # The current version of the gem.
-  VERSION = "0.2.1"
+  VERSION = "0.2.2"
 end

data/lib/boxcars.rb

@@ -107,6 +107,7 @@ module Boxcars
   end
 
   # Logging system
+  # debug log
   def self.debug(msg, color = nil, **options)
     msg = colorize(msg.to_s, color, **options) if color
     if logger
@@ -116,6 +117,7 @@ module Boxcars
     end
   end
 
+  # info log
   def self.info(msg, color = nil, **options)
     msg = colorize(msg.to_s, color, **options) if color
     if logger
@@ -125,6 +127,7 @@ module Boxcars
     end
   end
 
+  # warn log
   def self.warn(msg, color = nil, **options)
     msg = colorize(msg.to_s, color, **options) if color
     if logger
@@ -134,6 +137,7 @@ module Boxcars
     end
   end
 
+  # error log
   def self.error(msg, color = nil, **options)
     msg = colorize(msg.to_s, color, **options) if color
     if logger
@@ -143,6 +147,7 @@ module Boxcars
     end
   end
 
+  # simple colorization
   def self.colorize(str, color, **options)
     background = options[:background] || options[:bg] || false
     style = options[:style]
@@ -157,6 +162,8 @@ end
 
 require "boxcars/version"
 require "boxcars/prompt"
+require "boxcars/conversation_prompt"
+require "boxcars/conversation"
 require "boxcars/generation"
 require "boxcars/ruby_repl"
 require "boxcars/engine"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: boxcars
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors:
 - Francis Sullivan
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-03-08 00:00:00.000000000 Z
+date: 2023-03-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: debug
@@ -109,6 +109,8 @@ files:
 - lib/boxcars/boxcar/engine_boxcar.rb
 - lib/boxcars/boxcar/google_search.rb
 - lib/boxcars/boxcar/sql.rb
+- lib/boxcars/conversation.rb
+- lib/boxcars/conversation_prompt.rb
 - lib/boxcars/engine.rb
 - lib/boxcars/engine/engine_result.rb
 - lib/boxcars/engine/openai.rb