foobara-agent 0.0.7 → 0.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 968399c2d338c8f069712d0ad9437b928fe48bc7356e88fd86b9ff6fb7ae235f
-  data.tar.gz: 8483b856faf2f559985550cc631764121797f88963c0baf428b7f16646bd8b59
+  metadata.gz: 12e8facdeb95a000d81ec0ba8a133affa1ea66e470115e978bdc81f05bc51855
+  data.tar.gz: 294064df166d152c037db13263863b8ccfe32f064a78e77f6f5fe59af0be3829
 SHA512:
-  metadata.gz: 79be2912847cb407d1ad65166313c8b799b981a2b0da6f8c8e3f8132049c1eb67ec56a9307fdf887f8e659299883f0da4517762abc885fdf8b69949f869f54de
-  data.tar.gz: 169c02a8070d5feb4d307b41ff757294ca5a2e3b810ac0766cece386a357db2cf85075ae9ecb1f5ee2e0b1ec92c855eb202ee4166102a88341b8e5b4268eeb92
+  metadata.gz: 2c7fcaf044ba9bc821dcc42f9b0fb44b412fb42f9ef77dd4b7f3ed13cb8293cdb60d448d136e4e0567d104ecb022a00accfd05dbcf955e38185884d34e59668e
+  data.tar.gz: 380d9ae749b6a942b4e33b112f5aae35fd9600fa9ffcaf462fc0a35122354aae0658378190a4b9db3c0d65d2a12c9d49305b1d03a2881ce504d6d268cf039255

data/CHANGELOG.md

@@ -1,6 +1,14 @@
+## [0.0.8] - 2025-06-27
+
+- Improve what is logged and its formatting when verbose
+- Improve/experiment with what is stored in the command log
+- Experiment with Atoms instead of Aggregates
+- Add max_llm_calls_per_minute option
+
 ## [0.0.7] - 2025-06-23
 
-- 
+- Improvements to result type handling
+- Eliminate DescribeType command
 
 ## [0.0.6] - 2025-06-19
 

data/lib/foobara/agent.rb

@@ -12,7 +12,6 @@ module Foobara
       def reset_all
         [
           DetermineInputsForNextCommand,
-          DetermineNextCommand,
           NotifyUserThatCurrentGoalHasBeenAccomplished
         ].each do |command_class|
           command_class.clear_subclass_cache

data/src/foobara/agent/accomplish_goal.rb

@@ -3,6 +3,9 @@ require_relative "list_commands"
 module Foobara
   class Agent < CommandConnector
     class AccomplishGoal < Foobara::Command
+      # Using a const here so we can stub it in the test suite to speed things up
+      SECONDS_PER_MINUTE = 60
+
       possible_error :gave_up, context: { reason: :string }, message: "Gave up."
       possible_error :too_many_command_calls,
                      context: { maximum_command_calls: :integer }
@@ -26,19 +29,13 @@ module Foobara
                   one_of: Foobara::Ai::AnswerBot::Types::ModelEnum,
                   default: "claude-3-7-sonnet-20250219",
                   description: "The model to use for the LLM"
-        log_successful_determine_command_and_inputs_outcomes(
-          :boolean,
-          default: true,
-          description: "You can experiment with turning this off " \
-                       "if you want to see what happens if we don't log " \
-                       "successful command/input selection outcomes"
-        )
         choose_next_command_and_next_inputs_separately :boolean,
                                                        default: false,
                                                        description:
                                                          "By default, asks for next command and inputs together. " \
                                                          "You can experiment with getting the separately " \
                                                          "with this flag if you wish."
+        max_llm_calls_per_minute :integer, :allow_nil
       end
 
       result do
@@ -51,13 +48,16 @@ module Foobara
       def execute
         build_initial_context_if_necessary
 
+        simulate_describe_list_commands_command
         simulate_list_commands_run
-        simulate_describe_command_run_for_all_commands
+        # simulate_describe_command_run_for_all_commands
 
         until mission_accomplished or given_up
           increment_command_calls
           check_if_too_many_calls
 
+          throttle_llm_calls_if_necessary
+
           if choose_next_command_and_next_inputs_separately?
             determine_next_command_then_inputs_separately
           else
@@ -75,8 +75,8 @@ module Foobara
         build_result
       end
 
-      attr_accessor :context, :next_command_name, :next_command_inputs, :mission_accomplished, :given_up,
-                    :next_command_class, :next_command, :command_outcome, :timed_out,
+      attr_accessor :context, :next_command_name, :next_command_inputs, :next_command_raw_inputs, :mission_accomplished,
+                    :given_up, :next_command_class, :next_command, :command_outcome, :timed_out,
                     :final_result, :final_message, :command_response, :delayed_command_name,
                     :command_calls
 
@@ -86,9 +86,8 @@ module Foobara
       end
 
       def simulate_list_commands_run
-        return unless context.command_log.empty?
-
         self.next_command_name = ListCommands.full_command_name
+        self.next_command_raw_inputs = nil
         self.next_command_inputs = nil
         fetch_next_command_class
 
@@ -96,7 +95,19 @@ module Foobara
         log_last_command_outcome
       end
 
+      def simulate_describe_list_commands_command
+        self.next_command_name = DescribeCommand.full_command_name
+        self.next_command_inputs = { command_name: ListCommands.full_command_name }
+        self.next_command_raw_inputs = next_command_inputs
+        fetch_next_command_class
+
+        run_next_command
+        log_last_command_outcome
+      end
+
       def simulate_describe_command_run_for_all_commands
+        # TODO: currently not using this code path. Unclear if it is worth it.
+        # :nocov:
         return if context.command_log.size > 1
 
         ListCommands.run!(command_connector: agent)[:user_provided_commands].each do |full_command_name|
@@ -104,24 +115,26 @@ module Foobara
 
           self.next_command_name = DescribeCommand.full_command_name
           self.next_command_inputs = { command_name: full_command_name }
+          self.next_command_raw_inputs = next_command_inputs
           fetch_next_command_class
 
           run_next_command
           log_last_command_outcome
         end
+        # :nocov:
       end
 
       def determine_next_command_and_inputs(retries = 2)
         inputs_for_determine = {
           goal:,
           context:,
-          llm_model:,
-          command_class_names: all_command_classes
+          llm_model:
         }
 
         determine_command = DetermineNextCommandNameAndInputs.new(inputs_for_determine)
 
         outcome = begin
+          record_llm_call_timestamp
           determine_command.run
         rescue CommandPatternImplementation::Concerns::Result::CouldNotProcessResult => e
           # :nocov:
@@ -132,6 +145,7 @@ module Foobara
         if outcome.success?
           self.next_command_name = outcome.result[:command_name]
           self.next_command_inputs = outcome.result[:inputs]
+          self.next_command_raw_inputs = next_command_inputs
 
           outcome = validate_next_command_name
 
@@ -141,32 +155,25 @@ module Foobara
             if next_command_has_inputs?
               outcome = validate_next_command_inputs
 
-              if outcome.success?
-                if log_successful_determine_command_and_inputs_outcomes?
-                  log_command_outcome(
-                    command: determine_command,
-                    inputs: determine_command.inputs.except(:context)
-                  )
-                end
-              else
+              unless outcome.success?
                 log_command_outcome(
-                  command: determine_command,
-                  inputs: determine_command.inputs.except(:context),
-                  outcome:,
-                  result: outcome.result || determine_command.raw_result
+                  command_name: next_command_name,
+                  inputs: next_command_inputs,
+                  outcome:
                 )
 
                 determine_next_command_inputs
               end
             else
               self.next_command_inputs = {}
+              self.next_command_raw_inputs = next_command_inputs
             end
           else
             log_command_outcome(
-              command: determine_command,
-              inputs: determine_command.inputs&.except(:context),
+              command_name: next_command_name,
+              inputs: next_command_inputs,
               outcome:,
-              result: outcome.result || determine_command.raw_result
+              result: nil
             )
 
             if retries > 0
@@ -204,10 +211,23 @@ module Foobara
       end
 
       def validate_next_command_name
+        if next_command_name.is_a?(::String)
+          command_class = agent.transformed_command_from_name(next_command_name)
+
+          if command_class
+            self.next_command_name = command_class.full_command_name
+
+            return Outcome.success(next_command_name)
+          end
+        end
+
         outcome = command_name_type.process_value(next_command_name)
 
         if outcome.success?
+          # TODO: figure out a way to hit this path in the test suite
+          # :nocov:
           self.next_command_name = outcome.result
+          # :nocov:
         end
 
         outcome
@@ -231,17 +251,14 @@ module Foobara
                                    self.delayed_command_name = nil
                                    name
                                  else
-                                   command_class = DetermineNextCommand.for(
-                                     command_class_names: all_command_classes, agent_id: agent_name
-                                   )
-
                                    inputs = { goal:, context: }
                                    if llm_model
                                      inputs[:llm_model] = llm_model
                                    end
 
-                                   command = command_class.new(inputs)
+                                   command = DetermineNextCommand.new(inputs)
                                    outcome = begin
+                                     record_llm_call_timestamp
                                      command.run
                                    rescue CommandPatternImplementation::Concerns::Result::CouldNotProcessResult => e
                                      # :nocov:
@@ -250,12 +267,24 @@ module Foobara
                                    end
 
                                    if outcome.success?
-                                     if log_successful_determine_command_and_inputs_outcomes?
+                                     self.next_command_name = outcome.result
+
+                                     outcome = validate_next_command_name
+
+                                     unless outcome.success?
+                                       # TODO: figure out a way to hit this path in the test suite or delete it
+                                       # :nocov:
                                        log_command_outcome(
                                          command:,
                                          inputs: command.inputs.except(:context),
-                                         outcome:
+                                         outcome:,
+                                         result: outcome.result || command.raw_result
                                        )
+
+                                       if retries > 0
+                                         return determine_next_command_name(retries - 1)
+                                       end
+                                       # :nocov:
                                      end
                                    else
                                      # TODO: either figure out a way to hit this path in the test suite or delete it
@@ -303,6 +332,7 @@ module Foobara
 
                                      command = command_class.new(inputs)
                                      outcome = begin
+                                       record_llm_call_timestamp
                                        command.run
                                      rescue CommandPatternImplementation::Concerns::Result::CouldNotProcessResult => e
                                        # :nocov:
@@ -310,23 +340,15 @@ module Foobara
                                        # :nocov:
                                      end
 
-                                     if outcome.success?
-                                       if log_successful_determine_command_and_inputs_outcomes?
-                                         log_command_outcome(
-                                           command:,
-                                           inputs: command.inputs.except(:context),
-                                           outcome:
-                                         )
-                                       end
-                                     else
+                                     unless outcome.success?
                                        # TODO: either figure out a way to hit this path in the test suite or delete it
                                        # :nocov:
                                        log_command_outcome(
-                                         command:,
-                                         inputs: command.inputs.except(:context),
-                                         outcome:,
-                                         result: outcome.result || command.raw_result
+                                         command_name: next_command_name,
+                                         inputs: command.raw_result,
+                                         outcome:
                                        )
+
                                        if retries > 0
                                          return determine_next_command_inputs(retries - 1)
                                        end
@@ -351,7 +373,20 @@ module Foobara
 
       def run_next_command
         if verbose?
-          (io_out || $stdout).puts "Running #{next_command_name} with #{next_command_inputs}"
+          args = if next_command_inputs.nil? || next_command_inputs.empty?
+                   ""
+                 else
+                   s = next_command_inputs.to_s
+
+                   if s =~ /\A\{(.*)}\z/
+                     "(#{::Regexp.last_match(1)})"
+                   else
+                     # :nocov:
+                     raise "Unexpected next_command_inputs: #{next_command_inputs}"
+                     # :nocov:
+                   end
+                 end
+          (io_out || $stdout).puts "#{next_command_name}.run#{args}"
         end
 
         self.command_response = agent.run(
@@ -363,9 +398,7 @@ module Foobara
         self.command_outcome = command_response.outcome
 
         if verbose?
-          if command_outcome.success?
-            (io_out || $stdout).puts "Command #{command_response.command.class.full_command_name} succeeded"
-          else
+          unless command_outcome.success?
             # :nocov:
             (io_err || $stderr).puts(
               "Command #{command_response.command.class.full_command_name} failed #{command_outcome.errors_hash}"
@@ -398,8 +431,11 @@ module Foobara
       def log_command_outcome(command: nil, command_name: nil, inputs: nil, outcome: nil, result: nil)
         if command
           command_name ||= command.class.full_command_name
-          inputs ||= command.inputs
+          inputs ||= command.raw_inputs
           outcome ||= command.outcome
+        end
+
+        if outcome
           result ||= outcome.result
         end
 
@@ -457,10 +493,6 @@ module Foobara
         type.extends_type?(BuiltinTypes[:attributes]) && type.element_types.empty?
       end
 
-      def log_successful_determine_command_and_inputs_outcomes?
-        log_successful_determine_command_and_inputs_outcomes
-      end
-
       def choose_next_command_and_next_inputs_separately?
         choose_next_command_and_next_inputs_separately
       end
@@ -472,6 +504,52 @@ module Foobara
       def verbose?
         verbose
       end
+
+      def llm_call_timestamps
+        @llm_call_timestamps ||= []
+      end
+
+      attr_writer :llm_call_timestamps
+
+      def record_llm_call_timestamp
+        llm_call_timestamps.unshift(Time.now)
+      end
+
+      def llm_calls_in_last_minute
+        llm_call_timestamps.select { |t| t > (Time.now - 60) }
+      end
+
+      def llm_call_count_in_last_minute
+        llm_calls_in_last_minute.size
+      end
+
+      def time_until_llm_call_count_in_last_minute_changes
+        calls = llm_calls_in_last_minute
+
+        first_to_expire = calls.first
+
+        if first_to_expire
+          (first_to_expire + SECONDS_PER_MINUTE) - Time.now
+        else
+          # TODO: figure out how to test this code path
+          # :nocov:
+          0
+          # :nocov:
+        end
+      end
+
+      def throttle_llm_calls_if_necessary
+        return unless max_llm_calls_per_minute && max_llm_calls_per_minute > 0
+
+        if llm_call_count_in_last_minute >= max_llm_calls_per_minute
+          seconds = time_until_llm_call_count_in_last_minute_changes
+          if verbose?
+            (io_out || $stdout).puts "Sleeping for #{seconds} seconds to avoid LLM calls per minute limit"
+          end
+
+          sleep seconds
+        end
+      end
     end
   end
 end

data/src/foobara/agent/determine_inputs_for_next_command.rb

@@ -26,13 +26,20 @@ module Foobara
             end
 
             if command_class.inputs_type
-              klass.result command_class.inputs_type
+              transformer = CommandConnectors::Transformers::EntityToPrimaryKeyInputsTransformer.new(
+                to: command_class.inputs_type
+              )
+              klass.result transformer.from_type
             end
 
             klass
           end
         end
       end
+
+      def association_depth
+        Foobara::JsonSchemaGenerator::AssociationDepth::ATOM
+      end
     end
   end
 end

data/src/foobara/agent/determine_next_command.rb

@@ -3,45 +3,30 @@ require "foobara/llm_backed_command"
 module Foobara
   class Agent < CommandConnector
     class DetermineNextCommand < Foobara::LlmBackedCommand
-      extend Concerns::SubclassCacheable
+      description "Accepts the current goal, which might already be accomplished, " \
+                  "and context of the work  " \
+                  "so far and returns the name of " \
+                  "the next command to run to make progress towards " \
+                  "accomplishing the goal. If the goal has already been accomplished then choose the " \
+                  "NotifyUserThatCurrentGoalHasBeenAccomplished command."
 
-      class << self
-        # Allows us to give a more meaningful result type
-        def for(command_class_names:, agent_id:)
-          cached_subclass(agent_id) do
-            command_name = "Foobara::Agent::#{agent_id}::DetermineNextCommand"
-            klass = Util.make_class_p(command_name, self)
-
-            klass.description "Accepts the current goal, which might already be accomplished, " \
-                              "and context of the work  " \
-                              "so far and returns the name of " \
-                              "the next command to run to make progress towards " \
-                              "accomplishing the goal. If the goal has already been accomplished then choose the " \
-                              "NotifyUserThatCurrentGoalHasBeenAccomplished command."
-
-            klass.inputs do
-              goal :string, :required, "The current goal to accomplish. If the goal has already been accomplished " \
-                                       "by the previous command runs then choose " \
-                                       "NotifyUserThatCurrentGoalHasBeenAccomplished to stop the loop."
-              context Context, :required, "Context of progress so far"
-              llm_model :string,
-                        one_of: Foobara::Ai::AnswerBot::Types::ModelEnum,
-                        default: "claude-3-7-sonnet-20250219",
-                        description: "The model to use for the LLM"
-            end
-
-            klass.result :string,
-                         one_of: command_class_names,
-                         description: "Name of the next command to run to make progress " \
-                                      "towards accomplishing the mission"
-
-            klass
-          end
-        end
+      inputs do
+        goal :string, :required, "The current goal to accomplish. If the goal has already been accomplished " \
+                                 "by the previous command runs then choose " \
+                                 "NotifyUserThatCurrentGoalHasBeenAccomplished to stop the loop."
+        context Context, :required, "Context of progress so far"
+        llm_model :string,
+                  one_of: Foobara::Ai::AnswerBot::Types::ModelEnum,
+                  default: "claude-3-7-sonnet-20250219",
+                  description: "The model to use for the LLM"
       end
 
-      description "Accepts a goal and context of the work so far and returns the name of the next command to run to " \
-                  "make progress towards accomplishing the mission."
+      result :string,
+             description: "Name of the next command to run to make progress " \
+                          "towards accomplishing the mission"
+      def association_depth
+        Foobara::JsonSchemaGenerator::AssociationDepth::ATOM
+      end
     end
   end
 end

data/src/foobara/agent/determine_next_command_name_and_inputs.rb

@@ -14,7 +14,6 @@ module Foobara
                                  "by the previous command runs then choose " \
                                  "NotifyUserThatCurrentGoalHasBeenAccomplished to stop the loop."
         context Context, :required, "Context of the progress towards the goal so far"
-        command_class_names [:string], :required
         llm_model :string,
                   one_of: Foobara::Ai::AnswerBot::Types::ModelEnum,
                   default: "claude-3-7-sonnet-20250219",
@@ -25,6 +24,10 @@ module Foobara
         command_name :string, :required
         inputs :attributes, :allow_nil
       end
+
+      def association_depth
+        Foobara::JsonSchemaGenerator::AssociationDepth::ATOM
+      end
     end
   end
 end

data/src/foobara/agent.rb

@@ -22,7 +22,8 @@ module Foobara
                   :agent_commands_connected,
                   :verbose,
                   :io_out,
-                  :io_err
+                  :io_err,
+                  :max_llm_calls_per_minute
 
     def initialize(
       context: nil,
@@ -34,6 +35,7 @@ module Foobara
       verbose: false,
       io_out: nil,
       io_err: nil,
+      max_llm_calls_per_minute: nil,
       **opts
     )
       # TODO: shouldn't have to pass command_log here since it has a default, debug that
@@ -45,12 +47,19 @@ module Foobara
       self.verbose = verbose
       self.io_out = io_out
       self.io_err = io_err
-
-      unless opts.key?(:default_serializers)
-        opts = opts.merge(default_serializers: [
-                            Foobara::CommandConnectors::Serializers::ErrorsSerializer,
-                            Foobara::CommandConnectors::Serializers::AggregateSerializer
-                          ])
+      self.max_llm_calls_per_minute = max_llm_calls_per_minute
+
+      # unless opts.key?(:default_serializers)
+      #   opts = opts.merge(default_serializers: [
+      #                       Foobara::CommandConnectors::Serializers::ErrorsSerializer,
+      #                       Foobara::CommandConnectors::Serializers::AggregateSerializer
+      #                     ])
+      # end
+
+      unless opts.key?(:default_pre_commit_transformers)
+        opts = opts.merge(
+          default_pre_commit_transformers: Foobara::CommandConnectors::Transformers::LoadAtomsPreCommitTransformer
+        )
       end
 
       super(**opts)
@@ -73,9 +82,9 @@ module Foobara
       inputs_transformers = Util.array(inputs_transformers)
       inputs_transformers << CommandConnectors::Transformers::EntityToPrimaryKeyInputsTransformer
 
-      unless opts.key?(:aggregate_entities)
-        opts = opts.merge(aggregate_entities: true)
-      end
+      # unless opts.key?(:aggregate_entities)
+      #   opts = opts.merge(aggregate_entities: true)
+      # end
 
       super(*args, **opts.merge(inputs_transformers:))
     end
@@ -169,6 +178,10 @@ module Foobara
           inputs[:include_message_to_user_in_result] = include_message_to_user_in_result
         end
 
+        if max_llm_calls_per_minute && max_llm_calls_per_minute > 0
+          inputs[:max_llm_calls_per_minute] = max_llm_calls_per_minute
+        end
+
         self.current_accomplish_goal_command = AccomplishGoal.new(inputs)
 
         current_accomplish_goal_command.run.tap do |outcome|

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: foobara-agent
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.0.8
 platform: ruby
 authors:
 - Miles Georgi