boxcars 0.1.1 → 0.1.3

data/lib/boxcars/engine/openai.rb

@@ -1,18 +1,22 @@
 # frozen_string_literal: true
 
+require 'openai'
 # Boxcars is a framework for running a series of tools to get an answer to a question.
 module Boxcars
   # A engine that uses OpenAI's API.
   class Openai < Engine
     attr_reader :prompts, :open_ai_params, :model_kwargs, :batch_size
 
+    # The default parameters to use when asking the engine.
     DEFAULT_PARAMS = {
       model: "text-davinci-003",
       temperature: 0.7,
       max_tokens: 256
     }.freeze
 
+    # the default name of the engine
     DEFAULT_NAME = "OpenAI engine"
+    # the default description of the engine
     DEFAULT_DESCRIPTION = "useful for when you need to use AI to answer questions. " \
                           "You should ask targeted questions"
 
@@ -30,7 +34,9 @@ module Boxcars
     end
 
     # Get an answer from the engine.
-    # @param question [String] The question to ask the engine.
+    # @param prompt [String] The prompt to use when asking the engine.
+    # @param openai_access_token [String] The access token to use when asking the engine.
+    #   Defaults to Boxcars.configuration.openai_access_token.
     # @param kwargs [Hash] Additional parameters to pass to the engine if wanted.
     def client(prompt:, openai_access_token: 'not set', **kwargs)
       access_token = Boxcars.configuration.openai_access_token(openai_access_token: openai_access_token)
@@ -51,15 +57,20 @@ module Boxcars
     end
 
     # Build extra kwargs from additional params that were passed in.
+    # @param values [Hash] The values to build extra kwargs from.
     def build_extra(values:)
       values[:model_kw_args] = @open_ai_params.merge(values)
       values
     end
 
+    # Get the default parameters for the engine.
     def default_params
       open_ai_params
     end
 
+    # Get generation informaton
+    # @param sub_choices [Array<Hash>] The choices to get generation info for.
+    # @return [Array<Generation>] The generation information.
     def generation_info(sub_choices)
       sub_choices.map do |choice|
         Generation.new(
@@ -73,18 +84,9 @@ module Boxcars
     end
 
     # Call out to OpenAI's endpoint with k unique prompts.
-
-    #   Args:
-    #     prompts: The prompts to pass into the model.
-    #     stop: Optional list of stop words to use when generating.
-
-    #   Returns:
-    #     The full engine output.
-
-    #   Example:
-    #     .. code-block:: ruby
-
-    #       response = openai.generate(["Tell me a joke."])
+    # @param prompts [Array<String>] The prompts to pass into the model.
+    # @param stop [Array<String>] Optional list of stop words to use when generating.
+    # @return [EngineResult] The full engine output.
     def generate(prompts:, stop: nil)
       params = {}
       params[:stop] = stop if stop
@@ -112,21 +114,25 @@ module Boxcars
     # rubocop:enable Metrics/AbcSize
   end
 
+  # the identifying parameters for the engine
   def identifying_params
     params = { model_name: model_name }
     params.merge!(default_params)
     params
   end
 
+  # the engine type
   def engine_type
     "openai"
   end
 
   # calculate the number of tokens used
   def get_num_tokens(text:)
-    text.split.length
+    text.split.length # TODO: hook up to token counting gem
   end
 
+  # lookup the context size for a model by name
+  # @param modelname [String] The name of the model to lookup.
   def modelname_to_contextsize(modelname)
     model_lookup = {
       'text-davinci-003': 4097,
@@ -140,14 +146,10 @@ module Boxcars
   end
 
   # Calculate the maximum number of tokens possible to generate for a prompt.
-
-  #   Args:
-  #     prompt: The prompt to use.
-
-  #   Returns:
-  #     The maximum number of tokens possible to generate for a prompt.
-  def max_tokens_for_prompt(prompt)
-    num_tokens = get_num_tokens(prompt)
+  # @param prompt_text [String] The prompt text to use.
+  # @return [Integer] the number of tokens possible to generate.
+  def max_tokens_for_prompt(prompt_text)
+    num_tokens = get_num_tokens(prompt_text)
 
     # get max context size for model by name
     max_size = modelname_to_contextsize(model_name)

data/lib/boxcars/ruby_repl.rb

@@ -3,6 +3,8 @@
 module Boxcars
   # used by Boxcars to run ruby code
   class RubyREPL
+    # Execute ruby code
+    # @param code [String] The code to run
     def call(code:)
       puts "RubyREPL: #{code}".colorize(:red)
       output = ""
@@ -15,6 +17,8 @@ module Boxcars
       output
     end
 
+    # Execute ruby code
+    # @param command [String] The code to run
     def run(command)
       call(code: command)
     end

data/lib/boxcars/{conductor/conductor_action.rb → train/train_action.rb}

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
 module Boxcars
-  # Conductor's action to take.
-  class ConductorAction
+  # Train's action to take.
+  class TrainAction
     attr_accessor :boxcar, :boxcar_input, :log
 
     def initialize(boxcar: nil, boxcar_input: nil, log: nil)

data/lib/boxcars/{conductor/conductor_finish.rb → train/train_finish.rb}

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
 module Boxcars
-  # Conductor's return value
-  class ConductorFinish
+  # Train's return value
+  class TrainFinish
     attr_accessor :return_values, :log
 
     def initialize(return_values, log:)

data/lib/boxcars/{conductor → train}/zero_shot.rb

@@ -1,10 +1,13 @@
 # Agent for the MRKL chain
 module Boxcars
-  # A Conductor using the zero-shot react method.
-  class ZeroShot < Conductor
+  # A Train using the zero-shot react method.
+  class ZeroShot < Train
     attr_reader :boxcars, :observation_prefix, :engine_prefix
 
+    # default prompt prefix
     PREFIX = "Answer the following questions as best you can. You have access to the following actions:".freeze
+
+    # default prompt instructions
     FORMAT_INSTRUCTIONS = <<~FINPUT.freeze
       Use the following format:
 
@@ -18,6 +21,7 @@ module Boxcars
       Final Answer: the final answer to the original input question
     FINPUT
 
+    # default prompt suffix
     SUFFIX = <<~SINPUT.freeze
       Begin!
 
@@ -25,7 +29,11 @@ module Boxcars
       Thought:%<agent_scratchpad>s
     SINPUT
 
-    def initialize(boxcars:, engine:, name: 'Zero Shot', description: 'Zero Shot Conductor')
+    # @param boxcars [Array<Boxcars::Boxcar>] The boxcars to run.
+    # @param engine [Boxcars::Engine] The engine to use for this train.
+    # @param name [String] The name of the train. Defaults to 'Zero Shot'.
+    # @param description [String] The description of the train. Defaults to 'Zero Shot Train'.
+    def initialize(boxcars:, engine: nil, name: 'Zero Shot', description: 'Zero Shot Train')
       @observation_prefix = 'Observation: '
       @engine_prefix = 'Thought:'
       prompt = self.class.create_prompt(boxcars: boxcars)
@@ -33,16 +41,11 @@ module Boxcars
     end
 
     # Create prompt in the style of the zero shot agent.
-
-    #   Args:
-    #     boxcars: List of boxcars the agent will have access to, used to format the prompt.
-    #     prefix: String to put before the list of boxcars.
-    #     suffix: String to put after the list of boxcars.
-    #     input_variables: List of input variables the final prompt will expect.
-
-    #   Returns:
-    #     A Prompt with the template assembled from the pieces here.
-
+    # @param boxcars [Array<Boxcars::Boxcar>] List of boxcars the agent will have access to, used to format the prompt.
+    # @param prefix [String] String to put before the main prompt.
+    # @param suffix [String] String to put after the main prompt.
+    # @param input_variables [Array<Symbol>] List of input variables the final prompt will expect.
+    # @return [Boxcars::Prompt] A Prompt with the template assembled from the pieces here.
     def self.create_prompt(boxcars:, prefix: PREFIX, suffix: SUFFIX, input_variables: [:input, :agent_scratchpad])
       boxcar_strings = boxcars.map { |boxcar| "#{boxcar.name}: #{boxcar.description}" }.join("\n")
       boxcar_names = boxcars.map(&:name)
@@ -51,9 +54,12 @@ module Boxcars
       Prompt.new(template: template, input_variables: input_variables)
     end
 
+    # the final answer action string
     FINAL_ANSWER_ACTION = "Final Answer:".freeze
 
     # Parse out the action and input from the engine output.
+    # @param engine_output [String] The output from the engine.
+    # @return [Array<String>] The action and input.
     def get_action_and_input(engine_output:)
       # NOTE: if you're specifying a custom prompt for the ZeroShotAgent,
       #   you will need to ensure that it meets the following Regex requirements.
@@ -74,6 +80,9 @@ module Boxcars
       end
     end
 
+    # Extract the boxcar and input from the engine output.
+    # @param text [String] The output from the engine.
+    # @return [Array<Boxcars::Boxcar, String>] The boxcar and input.
     def extract_boxcar_and_input(text)
       get_action_and_input(engine_output: text)
     end

data/lib/boxcars/train.rb

@@ -0,0 +1,224 @@
+# frozen_string_literal: true
+
+module Boxcars
+  # @abstract
+  class Train < EngineBoxcar
+    attr_reader :engine, :boxcars, :name, :description, :prompt, :return_values, :return_intermediate_steps,
+                :max_iterations, :early_stopping_method
+
+    # A Train will use a engine to run a series of boxcars.
+    # @param engine [Boxcars::Engine] The engine to use for this train.
+    # @param boxcars [Array<Boxcars::Boxcar>] The boxcars to run.
+    # @param prompt [String] The prompt to use.
+    # @abstract
+    def initialize(boxcars:, prompt:, engine: nil, **kwargs)
+      @boxcars = boxcars
+      @name = name || self.class.name
+      @return_values = [:output]
+      @return_intermediate_steps = kwargs[:return_intermediate_steps] || false
+      @max_iterations = kwargs[:max_iterations]
+      @early_stopping_method = kwargs[:early_stopping_method] || "force"
+
+      super(prompt: prompt, engine: engine, name: kwargs[:name], description: kwargs[:description])
+    end
+
+    # Extract the boxcar name and input from the text.
+    # @param text [String] The text to extract from.
+    def extract_boxcar_and_input(text)
+    end
+
+    # the stop strings list
+    def stop
+      ["\n#{observation_prefix}"]
+    end
+
+    # build the scratchpad for the engine
+    # @param intermediate_steps [Array] The intermediate steps to build the scratchpad from.
+    # @return [String] The scratchpad.
+    def construct_scratchpad(intermediate_steps)
+      thoughts = ""
+      intermediate_steps.each do |action, observation|
+        thoughts += action.is_a?(String) ? action : action.log
+        thoughts += "\n#{observation_prefix}#{observation}\n#{engine_prefix}"
+      end
+      thoughts
+    end
+
+    # determine the next action
+    # @param full_inputs [Hash] The inputs to the engine.
+    # @return [Boxcars::Action] The next action.
+    def get_next_action(full_inputs)
+      full_output = predict(**full_inputs)
+      parsed_output = extract_boxcar_and_input(full_output)
+      while parsed_output.nil?
+        full_output = _fix_text(full_output)
+        full_inputs[:agent_scratchpad] += full_output
+        output = predict(**full_inputs)
+        full_output += output
+        parsed_output = extract_boxcar_and_input(full_output)
+      end
+      TrainAction.new(boxcar: parsed_output[0], boxcar_input: parsed_output[1], log: full_output)
+    end
+
+    # Given input, decided what to do.
+    # @param intermediate_steps [Array<Hash>] The intermediate steps taken so far along with observations.
+    # @param kwargs [Hash] User inputs.
+    # @return [Boxcars::Action] Action specifying what boxcar to use.
+    def plan(intermediate_steps, **kwargs)
+      thoughts = construct_scratchpad(intermediate_steps)
+      new_inputs = { agent_scratchpad: thoughts, stop: stop }
+      full_inputs = kwargs.merge(new_inputs)
+      action = get_next_action(full_inputs)
+      return TrainFinish.new({ output: action.boxcar_input }, log: action.log) if action.boxcar == finish_boxcar_name
+
+      action
+    end
+
+    # Prepare the agent for new call, if needed
+    def prepare_for_new_call
+    end
+
+    # Name of the boxcar to use to finish the chain
+    def finish_boxcar_name
+      "Final Answer"
+    end
+
+    # the input keys
+    # @return [Array<Symbol>] The input keys.
+    def input_keys
+      list = prompt.input_variables
+      list.delete(:agent_scratchpad)
+      list
+    end
+
+    # the output keys
+    def output_keys
+      return return_values + ["intermediate_steps"] if return_intermediate_steps
+
+      return_values
+    end
+
+    # should we continue to run?
+    # @param iterations [Integer] The number of iterations.
+    # @return [Boolean] Whether to continue.
+    def should_continue?(iterations)
+      return true if max_iterations.nil?
+
+      iterations < max_iterations
+    end
+
+    # handler before returning
+    # @param output [Boxcars::TrainFinish] The output.
+    # @param intermediate_steps [Array<Hash>] The intermediate steps.
+    # @return [Hash] The final output.
+    def pre_return(output, intermediate_steps)
+      puts output.log.colorize(:yellow)
+      final_output = output.return_values
+      final_output["intermediate_steps"] = intermediate_steps if return_intermediate_steps
+      final_output
+    end
+
+    # the prefix for the engine
+    # @param return_direct [Boolean] Whether to return directly.
+    # @return [String] The prefix.
+    def engine_prefix(return_direct)
+      return_direct ? "" : engine_prefix
+    end
+
+    # validate the prompt
+    # @param values [Hash] The values to validate.
+    # @return [Hash] The validated values.
+    # @raise [RuntimeError] If the prompt is invalid.
+    def validate_prompt(values:)
+      prompt = values["engine_chain"].prompt
+      unless prompt.input_variables.include?(:agent_scratchpad)
+        logger.warning("`agent_scratchpad` should be a variable in prompt.input_variables. Not found, adding it at the end.")
+        prompt.input_variables.append(:agent_scratchpad)
+        case prompt
+        when Prompt
+          prompt.template += "\n%<agent_scratchpad>s"
+        # when FewShotPromptTemplate
+        #   prompt.suffix += "\n%<agent_scratchpad>s"
+        else
+          raise ValueError, "Got unexpected prompt type #{type(prompt)}"
+        end
+      end
+      values
+    end
+
+    # get the stopped response
+    # @param early_stopping_method [String] The early stopping method.
+    # @param intermediate_steps [Array] The intermediate steps.
+    # @param kwargs [Hash] extra keword arguments.
+    # @return [Boxcars::Action] The action to take.
+    def return_stopped_response(early_stopping_method, intermediate_steps, **kwargs)
+      case early_stopping_method
+      when "force"
+        TrainFinish({ output: "Agent stopped due to max iterations." }, "")
+      when "generate"
+        thoughts = ""
+        intermediate_steps.each do |action, observation|
+          thoughts += action.log
+          thoughts += "\n#{observation_prefix}#{observation}\n#{engine_prefix}"
+        end
+        thoughts += "\n\nI now need to return a final answer based on the previous steps:"
+        new_inputs = { agent_scratchpad: thoughts, stop: _stop }
+        full_inputs = kwargs.merge(new_inputs)
+        full_output = predict(**full_inputs)
+        parsed_output = extract_boxcar_and_input(full_output)
+        if parsed_output.nil?
+          TrainFinish({ output: full_output }, full_output)
+        else
+          boxcar, boxcar_input = parsed_output
+          if boxcar == finish_boxcar_name
+            TrainFinish({ output: boxcar_input }, full_output)
+          else
+            TrainFinish({ output: full_output }, full_output)
+          end
+        end
+      else
+        raise "early_stopping_method should be one of `force` or `generate`, got #{early_stopping_method}"
+      end
+    end
+
+    # execute the train train
+    # @param inputs [Hash] The inputs.
+    # @return [Hash] The output.
+    def call(inputs:)
+      prepare_for_new_call
+      name_to_boxcar_map = boxcars.to_h { |boxcar| [boxcar.name, boxcar] }
+      intermediate_steps = []
+      iterations = 0
+      while should_continue?(iterations)
+        output = plan(intermediate_steps, **inputs)
+        return pre_return(output, intermediate_steps) if output.is_a?(TrainFinish)
+
+        if (boxcar = name_to_boxcar_map[output.boxcar])
+          begin
+            observation = boxcar.run(output.boxcar_input)
+            return_direct = boxcar.return_direct
+          rescue StandardError => e
+            puts "Error in #{boxcar.name} boxcar#call: #{e}".colorize(:red)
+            raise e
+          end
+        else
+          observation = "#{output.boxcar} is not a valid boxcar, try another one."
+          return_direct = false
+        end
+        puts "#Observation: #{observation}".colorize(:green)
+        intermediate_steps.append([output, observation])
+        if return_direct
+          output = TrainFinish.new({ return_values[0] => observation }, "")
+          return pre_return(output, intermediate_steps)
+        end
+        iterations += 1
+      end
+      output = return_stopped_response(early_stopping_method, intermediate_steps, **inputs)
+      pre_return(output, intermediate_steps)
+    end
+  end
+end
+
+require "boxcars/train/train_action"
+require "boxcars/train/train_finish"
+require "boxcars/train/zero_shot"

data/lib/boxcars/version.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
 module Boxcars
-  VERSION = "0.1.1"
+  # The current version of the gem.
+  VERSION = "0.1.3"
 end

data/lib/boxcars.rb

@@ -6,12 +6,21 @@ require 'logger'
 module Boxcars
   # Error class for all Boxcars errors.
   class Error < StandardError; end
+
+  # Error class for all Boxcars configuration errors.
   class ConfigurationError < Error; end
+
+  # Error class for all Boxcars argument errors.
   class ArgumentError < Error; end
+
+  # Error class for all Boxcars value errors.
   class ValueError < Error; end
 
   # simple string colorization
   class ::String
+    # colorize a string
+    # @param color [Symbol] The color to use.
+    # @param options [Hash] The options to use.
     def colorize(color, options = {})
       background = options[:background] || options[:bg] || false
       style = options[:style]
@@ -27,12 +36,13 @@ module Boxcars
   # Configuration contains gem settings
   class Configuration
     attr_writer :openai_access_token, :serpapi_api_key
-    attr_accessor :organization_id, :logger
+    attr_accessor :organization_id, :logger, :log_prompts
 
     def initialize
       @organization_id = nil
       @logger = Rails.logger if defined?(Rails)
       @logger ||= Logger.new($stdout)
+      @log_prompts = false
     end
 
     # @return [String] The OpenAI Access Token either from arg or env.
@@ -84,6 +94,16 @@ module Boxcars
   def self.configure
     yield(configuration)
   end
+
+  # Return the default Train class.
+  def self.default_train
+    Boxcars::ZeroShot
+  end
+
+  # Return the default Engine class.
+  def self.default_engine
+    Boxcars::Openai
+  end
 end
 
 require "boxcars/version"
@@ -92,4 +112,4 @@ require "boxcars/generation"
 require "boxcars/ruby_repl"
 require "boxcars/engine"
 require "boxcars/boxcar"
-require "boxcars/conductor"
+require "boxcars/train"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: boxcars
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.3
 platform: ruby
 authors:
 - Francis Sullivan
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-02-16 00:00:00.000000000 Z
+date: 2023-02-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: debug
@@ -81,7 +81,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
-description: You simply give a number of boxcars to a conductor, and it does the magic.
+description: You simply give a number of boxcars to a train, and it does the magic.
 email:
 - hi@boxcars.ai
 executables: []
@@ -106,17 +106,16 @@ files:
 - lib/boxcars/boxcar/engine_boxcar.rb
 - lib/boxcars/boxcar/serp.rb
 - lib/boxcars/boxcar/sql.rb
-- lib/boxcars/conductor.rb
-- lib/boxcars/conductor/conductor_action.rb
-- lib/boxcars/conductor/conductor_executer.rb
-- lib/boxcars/conductor/conductor_finish.rb
-- lib/boxcars/conductor/zero_shot.rb
 - lib/boxcars/engine.rb
 - lib/boxcars/engine/engine_result.rb
 - lib/boxcars/engine/openai.rb
 - lib/boxcars/generation.rb
 - lib/boxcars/prompt.rb
 - lib/boxcars/ruby_repl.rb
+- lib/boxcars/train.rb
+- lib/boxcars/train/train_action.rb
+- lib/boxcars/train/train_finish.rb
+- lib/boxcars/train/zero_shot.rb
 - lib/boxcars/version.rb
 homepage: https://github.com/BoxcarsAI/boxcars
 licenses:

data/lib/boxcars/conductor/conductor_executer.rb

@@ -1,96 +0,0 @@
-# frozen_string_literal: true
-
-module Boxcars
-  # Consists of an conductor using boxcars.
-  class ConductorExecuter < EngineBoxcar
-    attr_accessor :conductor, :boxcars, :return_intermediate_steps, :max_iterations, :early_stopping_method
-
-    # @param conductor [Boxcars::Conductor] The conductor to use.
-    # @param boxcars [Array<Boxcars::Boxcar>] The boxcars to use.
-    # @param return_intermediate_steps [Boolean] Whether to return the intermediate steps. Defaults to false.
-    # @param max_iterations [Integer] The maximum number of iterations to run. Defaults to nil.
-    # @param early_stopping_method [String] The early stopping method to use. Defaults to "force".
-    def initialize(conductor:, boxcars:, return_intermediate_steps: false, max_iterations: nil,
-                   early_stopping_method: "force")
-      @conductor = conductor
-      @boxcars = boxcars
-      @return_intermediate_steps = return_intermediate_steps
-      @max_iterations = max_iterations
-      @early_stopping_method = early_stopping_method
-      # def initialize(prompt:, engine:, output_key: "text", name: nil, description: nil)
-      super(prompt: conductor.prompt, engine: conductor.engine, name: conductor.name, description: conductor.description)
-    end
-
-    def same_boxcars?(boxcar_names)
-      conductor.allowed_boxcars.sort == boxcar_names
-    end
-
-    def validate_boxcars
-      boxcar_names = boxcars.map(&:name).sort
-      return if same_boxcars?(boxcar_names)
-
-      raise "Allowed boxcars (#{conductor.allowed_boxcars}) different than provided boxcars (#{boxcar_names})"
-    end
-
-    def input_keys
-      conductor.input_keys
-    end
-
-    def output_keys
-      return conductor.return_values + ["intermediate_steps"] if return_intermediate_steps
-
-      conductor.return_values
-    end
-
-    def should_continue?(iterations)
-      return true if max_iterations.nil?
-
-      iterations < max_iterations
-    end
-
-    # handler before returning
-    def pre_return(output, intermediate_steps)
-      puts output.log.colorize(:yellow)
-      final_output = output.return_values
-      final_output["intermediate_steps"] = intermediate_steps if return_intermediate_steps
-      final_output
-    end
-
-    def engine_prefix(return_direct)
-      return_direct ? "" : conductor.engine_prefix
-    end
-
-    def call(inputs:)
-      conductor.prepare_for_new_call
-      name_to_boxcar_map = boxcars.to_h { |boxcar| [boxcar.name, boxcar] }
-      intermediate_steps = []
-      iterations = 0
-      while should_continue?(iterations)
-        output = conductor.plan(intermediate_steps, **inputs)
-        return pre_return(output, intermediate_steps) if output.is_a?(ConductorFinish)
-
-        if (boxcar = name_to_boxcar_map[output.boxcar])
-          begin
-            observation = boxcar.run(output.boxcar_input)
-            return_direct = boxcar.return_direct
-          rescue StandardError => e
-            puts "Error in #{boxcar.name} boxcar#call: #{e}".colorize(:red)
-            raise e
-          end
-        else
-          observation = "#{output.boxcar} is not a valid boxcar, try another one."
-          return_direct = false
-        end
-        puts "#Observation: #{observation}".colorize(:green)
-        intermediate_steps.append([output, observation])
-        if return_direct
-          output = ConductorFinish.new({ conductor.return_values[0] => observation }, "")
-          return pre_return(output, intermediate_steps)
-        end
-        iterations += 1
-      end
-      output = conductor.return_stopped_response(early_stopping_method, intermediate_steps, **inputs)
-      pre_return(output, intermediate_steps)
-    end
-  end
-end