composable_agents 1.0.0

data/lib/composable_agents/mixins/resumable.rb

@@ -0,0 +1,214 @@
+require 'json'
+require 'fileutils'
+
+module ComposableAgents
+  module Mixins
+    # Mixin adding resumable step capabilities to agents.
+    # An agent prepending this mixin can use the following:
+    # * A new constructor named parameter run_id that identifies the run that can be resumable.
+    # * A step re-entrant method that defines a part of the agent's processing whose input/output is persisted
+    #   and that can be skipped if it was previously executed.
+    # * An agent_step method that calls a sub-agent with the artifacts and also tracks the state of this agent.
+    #   * Any agent that implements the methods export_state and import_state will benefit from its state's serialization automatically.
+    # * An instance variable @artifacts that stores artifacts (initialized with input ones) that are JSON serialized by steps.
+    # Artifacts used with this mixin, and states returned by used agents should be JSON-serializable.
+    # This mixin uses the following methods from the agent:
+    # - `#export_state -> Object` Optional method returning the current JSON-serializable state of the agent.
+    # - `#import_state(state)` Optional method that sets the agent state from a JSON-serializable object.
+    module Resumable
+      # @!group Public API
+
+      # Constructor
+      #
+      # @param run_id [String, nil] ID identifying this run to reuse previously executed steps, or nil if there is no resumability needed
+      def initialize(*args, run_id: nil, **kwargs)
+        super(*args, **kwargs)
+        @run_id = run_id
+      end
+
+      # @!group Internal
+
+      # Execute the agent to generate some output artifacts based on some input artifacts.
+      #
+      # @param input_artifacts [Hash{Symbol => Object}] The input artifacts content, per artifact name
+      # @return [Hash{Symbol => Object}] The output artifacts returned by the Proc
+      def run(**input_artifacts)
+        # The artifacts store, JSON serializable
+        @artifacts = input_artifacts.dup
+        # List of the levels' next step index, following the hierarchy of recusive step calls.
+        # This is only used if there is a persistent run ID.
+        # For example here are the values of this variable if we have this code:
+        # # @steps_idx == [0]
+        # step(:a) do
+        #   # @steps_idx == [0, 0]
+        #   step(:a1) do
+        #     # @steps_idx == [0, 0, 0]
+        #   end
+        #   # @steps_idx == [0, 1]
+        #   step(:a2) do
+        #     # @steps_idx == [0, 1, 0]
+        #     step(:a21) do
+        #       # @steps_idx == [0, 1, 0, 0]
+        #     end
+        #     # @steps_idx == [0, 1, 1]
+        #     step(:a22) do
+        #       # @steps_idx == [0, 1, 1, 0]
+        #     end
+        #     # @steps_idx == [0, 1, 2]
+        #   end
+        #   # @steps_idx == [0, 2]
+        # end
+        # # @steps_idx == [1]
+        # step(:b) do
+        #   # @steps_idx == [1, 0]
+        # end
+        @steps_idx = [0] unless @run_id.nil?
+        super
+      end
+
+      private
+
+      # Define a step that can be serialized and resumed.
+      # This will store the state of this step in the file system.
+      # If this step was already executed, skip it and update its artifacts from the file system store.
+      #
+      # @param step_name [Symbol] Step name.
+      # @param kwargs [Hash{Symbol => Object}] Additional input artifacts to merge before the step executes.
+      # @yield The code called for this step
+      def step(step_name = :step, **kwargs, &)
+        internal_step(step_name:, agent: nil, extra_input_artifacts: kwargs, &)
+      end
+
+      # Define a step that will just run an agent.
+      # This will use the artifacts store for input and output artifacts.
+      # Handle the context of the agent if needed.
+      #
+      # @param agent [Agent] The agent to run.
+      # @param kwargs [Hash{Symbol => Object}] Additional input artifacts to merge before the step executes.
+      def step_agent(agent, **kwargs)
+        internal_step(step_name: :"agent_run_#{(agent.name || 'unnamed').gsub(/[^\w]/, '_')}", agent:, extra_input_artifacts: kwargs) do
+          @artifacts.merge!(agent.run(**@artifacts))
+        end
+      end
+
+      # Below methods are not supposed to be used directly by the mixin user.
+      # They are only internals.
+
+      # Define a step that can be serialized and resumed.
+      # This will store the state of this step in the file system.
+      # If this step was already executed, skip it and update its artifacts from the file system store.
+      # Handle the state of an optional agent in the case this step is executed for an agent.
+      # This method should not be used directly.
+      #
+      # @param step_name [Symbol] Step name.
+      # @param agent [Agent, nil] Agent that is used in this step, or nil if none.
+      # @param extra_input_artifacts [Hash{Symbol => Object}] Additional input artifacts to merge before the step executes.
+      # @yield The code called for this step
+      def internal_step(step_name:, agent:, extra_input_artifacts: {})
+        @artifacts.merge!(extra_input_artifacts)
+        if @run_id.nil?
+          yield
+        else
+          # Compute the current step state
+          step_state = current_step_state(agent:)
+          # Read the persisted step state if any
+          step_full_name = "#{@steps_idx.join('-')}-#{step_name}"
+          saved_input_state, saved_output_state = saved_step_states(step_full_name)
+          # If the input exists, it means the step was already executed.
+          # If it is the same state as the current one, skip the step and set the current state to the stored output step state.
+          if step_state == saved_input_state
+            set_current_step_state(saved_output_state, agent:)
+            log_debug "[Step #{step_full_name}] - Already executed - Got #{@artifacts.size} artifacts from persistence: #{@artifacts.keys.join(', ')}"
+          else
+            # Clone state before yielding because it will certainly be modified
+            input_step_state = clone_step_state(step_state)
+            @steps_idx << 0
+            yield
+            @steps_idx.pop
+            store_step_states(step_full_name, input: input_step_state, output: current_step_state(agent:))
+            log_debug "[Step #{step_full_name}] - Executed - Stored #{@artifacts.size} artifacts in persistence: #{@artifacts.keys.join(', ')}"
+          end
+          @steps_idx[-1] += 1
+        end
+      end
+
+      # Get the current step state.
+      #
+      # @param agent [Agent, nil] Agent that is used in this step, or nil if none.
+      # @return [Hash{Symbol => Object}] The current step state
+      def current_step_state(agent:)
+        step_state = {
+          artifacts: @artifacts
+        }
+        step_state[:agent_state] = agent.export_state if !agent.nil? && agent.respond_to?(:export_state)
+        step_state
+      end
+
+      # Get saved step states JSON file for a given full step name.
+      #
+      # @param step_full_name [String] Step full name
+      # @return [String] Corresponding JSON file that stores the step states
+      def saved_step_states_json(step_full_name)
+        "#{@composable_agents_dir}/runs/#{@run_id}/#{step_full_name.gsub(/[^\w.]/, '_')}.json"
+      end
+
+      # Get saved step states from a given full step name.
+      # This will read the step states from the persistence layer.
+      #
+      # @param step_full_name [String] Step full name
+      # @return [Array<Hash{Symbol => Object}, nil>] The saved step states:
+      #   0. [Hash{Symbol => Object}, nil] The input step state or nil if none
+      #   1. [Hash{Symbol => Object}, nil] The output step state or nil if none
+      def saved_step_states(step_full_name)
+        step_json_file = saved_step_states_json(step_full_name)
+        step_info = File.exist?(step_json_file) ? JSON.parse(File.read(step_json_file)).transform_keys(&:to_sym) : {}
+        step_info[:input]&.transform_keys!(&:to_sym)
+        step_info.dig(:input, :artifacts)&.transform_keys!(&:to_sym)
+        step_info[:output]&.transform_keys!(&:to_sym)
+        step_info.dig(:output, :artifacts)&.transform_keys!(&:to_sym)
+        [step_info[:input], step_info[:output]]
+      end
+
+      # Store step states for a given full step name in the persistence layer.
+      #
+      # @param step_full_name [String] Step full name
+      # @param input [Hash{Symbol => Object}] The input step state
+      # @param output [Hash{Symbol => Object}] The output step state
+      def store_step_states(step_full_name, input:, output:)
+        step_json_file = saved_step_states_json(step_full_name)
+        FileUtils.mkdir_p(File.dirname(step_json_file))
+        File.write(
+          step_json_file,
+          JSON.dump(
+            {
+              input:,
+              output:
+            }
+          )
+        )
+      end
+
+      # Set the current state to a given step state
+      #
+      # @param step_state [Hash{Symbol => Object}] The step state to use
+      # @param agent [Agent, nil] Agent that is used in this step, or nil if none.
+      def set_current_step_state(step_state, agent:)
+        @artifacts = step_state[:artifacts]
+        agent.import_state(step_state[:agent_state]) if !agent.nil? && agent.respond_to?(:import_state)
+      end
+
+      # Clone a step state.
+      #
+      # @param step_state [Hash{Symbol => Object}] Step state to be cloned
+      # @return [Hash{Symbol => Object}] Cloned step state
+      def clone_step_state(step_state)
+        cloned_step_state = {
+          artifacts: step_state[:artifacts].dup
+        }
+        # Perform a deep clone using JSON
+        cloned_step_state[:agent_state] = JSON.parse(JSON.dump(step_state[:agent_state])) if step_state.key?(:agent_state)
+        cloned_step_state
+      end
+    end
+  end
+end

data/lib/composable_agents/mixins/user_interaction.rb

@@ -0,0 +1,42 @@
+module ComposableAgents
+  module Mixins
+    # Mixin that adds user interaction capabilities to PromptDrivenAgent agents.
+    # This defines the following methods:
+    # - `#ask(question) -> answer` Calls the `#answer_to` method to get the answer to a question and logs the conversation.
+    # An agent using this Mixin should define the private method `#answer_to`` to handle the question.
+    # Default handling is asking the question on the terminal.
+    module UserInteraction
+      # @!group Public API
+
+      # Answer an agent's question
+      #
+      # @param question [String] The agent's question
+      # @return [String] The answer that should be sent back to the agent
+      def ask(question)
+        track_message(message: question, author: "Agent #{full_name}", question: true)
+        answer = answer_to(question)
+        track_message(message: answer, author: 'User')
+        answer
+      end
+
+      private
+
+      # Answer an agent's question
+      #
+      # @param question [String] The agent's question
+      # @return [String] The answer that should be sent back to the agent
+      def answer_to(question)
+        if defined?(super)
+          super
+        else
+          # Provide a default way from terminal
+          puts
+          puts "Agent is asking a question:\n#{question}"
+          puts
+          puts 'Write answer and hit Enter...'
+          $stdin.gets.strip
+        end
+      end
+    end
+  end
+end

data/lib/composable_agents/prompt_driven_agent.rb

@@ -0,0 +1,241 @@
+require 'time'
+
+module ComposableAgents
+  # Agent implementation that uses a prompt rendering strategy to render prompts for a prompting engine.
+  # The following prompts are considered:
+  # - A system prompt, that defines the Agent's behaviour, or persona.
+  # - User prompts, that are used as user inputs guiding the agent.
+  # - Retry prompts, used to tell the Agent that the task at hand is still incomplete and needs more work.
+  #     For example when an artifact has not been created, and the agent needs to try it again.
+  #
+  # Prompt rendering strategies are useful because different prompt-driven agents would benefit from different
+  #   prompt formats or structures (JSON, Markdown, explicit ordered lists, in-lining artifacts' contents without tools...).
+  # This agent automatically records non-rendered conversation (prompts + outputs) in a `conversation` store, part of its state.
+  class PromptDrivenAgent < Agent
+    # @!group Public API
+
+    # @return [String, nil] Agent's role, or nil for the agent's default
+    attr_accessor :role
+
+    # @return [String, nil] Agent's objective, or nil for the agent's default
+    attr_accessor :objective
+
+    # @return [Object, nil] Agent's original instructions, or nil if no system instructions are needed (see Instructions#initialize)
+    attr_accessor :system_instructions
+
+    # @return [String, nil] Constraints to be respected, or nil for the agent's default
+    attr_accessor :constraints
+
+    # @return [Array<Hash{Symbol => Object}>] The conversation (user prompts, responses) that happened with this agent.
+    #   Each item of a conversation has the following properties:
+    #   - author [String] Author of the message
+    #   - at [String] UTC timestamp of the message at format YYYY-mm-dd HH:MM:SS
+    #   - message [String] The message itself
+    #   - question [Boolean] Is this message a question expecting a reply? Defaults to false.
+    attr_reader :conversation
+
+    # Initialize a new PromptDrivenAgent with the information needed for prompts and the selected prompt rendering strategy.
+    # If no name is provided, it will default to 'Executor'.
+    #
+    # @param role [String, nil] Agent's role, or nil for the agent's default
+    # @param objective [String, nil] Agent's objective, or nil for the agent's default
+    # @param system_instructions [Object, nil] Original instructions for the agent, or nil if no system instructions are needed.
+    #   The kind of instructions that can be given are defined by the Instructions's constructor (see Instructions#initialize).
+    # @param constraints [String, nil] Constraints to be respected, or nil for the agent's default
+    # @param strategy [Module] The prompt rendering strategy
+    def initialize(
+      *args,
+      role: nil,
+      objective: nil,
+      system_instructions: nil,
+      constraints: nil,
+      strategy: PromptRenderingStrategy::Markdown,
+      **kwargs
+    )
+      super(*args, **kwargs)
+      singleton_class.include strategy
+      @role = role
+      @objective = objective
+      @system_instructions = system_instructions
+      @constraints = constraints
+      @conversation = []
+    end
+
+    # Execute the agent to generate some output artifacts based on some input artifacts.
+    #
+    # @param user_instructions [Object, nil] Instructions for the user prompt, that will be rendered.
+    #   The kind of instructions that can be given are defined by the Instructions's constructor (see Instructions#initialize).
+    # @param input_artifacts [Hash{Symbol => Object}] The input artifacts content, per artifact name
+    # @return [Hash{Symbol => Object}] The output artifacts
+    def run(user_instructions: nil, **input_artifacts)
+      @input_artifacts = input_artifacts
+      @output_artifacts = {}
+      @output_artifacts_errors = {}
+      @system_prompt = render_system_prompt(render_instructions(@system_instructions))
+      log_debug "System prompt: #{@system_prompt}"
+      converse(user_instructions, input_artifacts: @input_artifacts, author: 'User')
+      if respond_to?(:normalized_output_artifacts_contracts, true)
+        # We know which output artifacts we are expecting.
+        # Therefore check if some are missing and prompt again if that's the case.
+        # TODO: Implement a max number of retries and throw an exception if it exceeds.
+        loop do
+          missing_artifacts = normalized_output_artifacts_contracts
+            .reject { |artifact_name, _artifact_description| @output_artifacts.key?(artifact_name) }
+            .to_h do |artifact_name, artifact_description|
+              [
+                artifact_name,
+                artifact_description.merge(@output_artifacts_errors[artifact_name] ? { error: @output_artifacts_errors[artifact_name] } : {})
+              ]
+            end
+          break if missing_artifacts.empty?
+
+          converse(missing_output_user_instructions(missing_artifacts))
+        end
+      end
+      @output_artifacts
+    end
+
+    # @!group Internal
+
+    # Define input artifacts contracts
+    #
+    # @return [Hash{Symbol => Object}] Set of input artifacts description, per artifact name
+    def input_artifacts_contracts
+      {
+        user_instructions: {
+          description: 'User instructions',
+          optional: true
+        }
+      }
+    end
+
+    # Define output artifacts contracts
+    #
+    # @return [Hash{Symbol => Object}>] Set of output artifacts description, per artifact name
+    def output_artifacts_contracts
+      {}
+    end
+
+    # Export the agent state for persistence
+    #
+    # @return [Object] Serialized state that can be marshalled to JSON
+    def export_state
+      deep_transform_keys(
+        {
+          conversation: @conversation.map do |message|
+            message.merge(at: message[:at].strftime('%F %T'))
+          end
+        },
+        &:to_s
+      )
+    end
+
+    # Import the agent state from persistence
+    #
+    # @param state [Object] Serialized state
+    def import_state(state)
+      @conversation = deep_transform_keys(state, &:to_sym)[:conversation].map do |message|
+        message.merge(at: Time.parse("#{message.delete(:at)} UTC"))
+      end
+    end
+
+    # Save an output artifact.
+    # This method can be used at anytime while prompting, when the agent is able to produce an output artifact.
+    #
+    # @param artifact_name [Symbol] Output artifact name
+    # @param content [Object] Output artifact content
+    def save_output_artifact(artifact_name, content)
+      @output_artifacts[artifact_name] = content
+      @output_artifacts_errors.delete(artifact_name)
+      log_debug "[Artifact] - Received output artifact #{artifact_name}"
+    end
+
+    # Report an error on an output artifact.
+    # This method can be used at anytime while prompting, when the agent is unable to produce an output artifact
+    #   because of an error that should be communicated back to the agent.
+    # Make sure previous versions of this output artifact are removed to not store wrong versions by mistake.
+    #
+    # @param artifact_name [Symbol] Output artifact name
+    # @param error [String] Error associated to this output artifact
+    def report_error_for_output_artifact(artifact_name, error)
+      @output_artifacts.delete(artifact_name)
+      @output_artifacts_errors[artifact_name] = error
+      # TODO: Make this as a warning message
+      log_debug "[Artifact] - Should have received content for output artifact `#{artifact_name}` " \
+        "but the following error occurred: #{error}"
+    end
+
+    private
+
+    # Render instructions using the prompt rendering strategy.
+    # Returns nil if instructions is nil.
+    #
+    # @param instructions [Object, nil] Instructions to render, or nil if none (see Instructions#initialize).
+    # @return [String, nil] The rendered instructions, or nil if none
+    def render_instructions(instructions)
+      return nil unless instructions
+
+      render_instructions_list(
+        Instructions.new(instructions).map do |instruction_type, instruction|
+          send(:"render_instruction_#{instruction_type}", instruction)
+        end
+      )
+    end
+
+    # Prompt a user prompt and record it with its response in the conversation.
+    #
+    # @param instructions [Object, nil] The instructions for the user prompt (see Instructions#initialize), or nil if none
+    # @param input_artifacts [Hash{Symbol => Object}] The input artifacts content, per artifact name
+    # @param author [String] Author of this message. Usually User if it is user input, but can be Orchestrator or anything else
+    def converse(instructions, input_artifacts: {}, author: 'Orchestrator')
+      rendered_instructions = render_instructions(instructions)
+      rendered_user_prompt = render_user_prompt(rendered_instructions, input_artifacts:)
+      log_debug "Rendered User prompt: #{rendered_user_prompt}"
+      track_message(message: rendered_instructions, author:)
+      response = prompt(rendered_user_prompt)
+      log_debug "Raw Agent #{full_name} response: #{response}"
+      track_message(message: response, author: "Agent #{full_name}")
+    end
+
+    # Process a user prompt.
+    #
+    # @param user_prompt [String] The rendered user prompt
+    # @return [String] The output of the prompt
+    def prompt(user_prompt)
+      raise NotImplementedError, 'This method should be implemented by a PromptDrivenAgent subclass'
+    end
+
+    # Track a message that is part of the conversation with this agent
+    #
+    # @param message [String, #to_hash, nil] The message content, as a String or an object that can be hashed, or nil if none.
+    # @param author [String] Author of the message.
+    # @param question [Boolean] Is this message a question expecting a reply?
+    def track_message(message:, author: 'Orchestrator', question: false)
+      @conversation << {
+        at: Time.now.utc,
+        author:,
+        message: message.is_a?(String) ? message : message&.to_hash,
+        question:
+      }
+    end
+
+    # Apply a deep nested transformation on a Hash's keys.
+    # Traverse nested arrays and hashes.
+    #
+    # @param obj [Object] The source object to transform (could be Hash, Array, or any other object).
+    # @yield [#call(key) -> Object] Transformation operation
+    # @yieldparam key [Object] Source key to be transformed
+    # @yieldreturn [Object] Transformed key
+    # @return [Object] The transformed object
+    def deep_transform_keys(obj, &)
+      case obj
+      when Hash
+        obj.each_with_object({}) { |(key, value), result| result[yield(key)] = deep_transform_keys(value, &) }
+      when Array
+        obj.map { |value| deep_transform_keys(value, &) }
+      else
+        obj
+      end
+    end
+  end
+end

data/lib/composable_agents/prompt_rendering_strategy/markdown.rb

@@ -0,0 +1,108 @@
+module ComposableAgents
+  # Collection of prompt rendering strategies that can be used with PromptDrivenAgent's agents
+  module PromptRenderingStrategy
+    # Render prompt as Markdown documents
+    module Markdown
+      # @!group Internal
+
+      # Render an instruction of type text
+      #
+      # @param instruction [String] The instruction to render
+      # @return [String] The rendered instruction
+      def render_instruction_text(instruction)
+        instruction
+      end
+
+      # Render an instruction of type ordered_list
+      #
+      # @param instruction [Array<String>] The instruction to render
+      # @return [String] The rendered instruction
+      def render_instruction_ordered_list(instruction)
+        instruction.map.with_index { |step, step_idx| "# #{step_idx + 1}. #{step}" }.join("\n\n")
+      end
+
+      # Render a list of rendered instructions
+      #
+      # @param instructions [Array<String>] The instructions list to render
+      # @return [String] The rendered instructions list
+      def render_instructions_list(instructions)
+        instructions.map { |instruction| Utils::Markdown.align_markdown_headers(instruction, level: 1).strip }.join("\n\n")
+      end
+
+      # Render the system prompt.
+      # The following instance variables are accessible to render the prompt:
+      # - `@input_artifacts`
+      # - `@role`
+      # - `@objective`
+      # - `@constraints`
+      #
+      # @param rendered_instructions [String, nil] The rendered system instructions, or nil if none
+      # @return [String] The rendered system prompt
+      def render_system_prompt(rendered_instructions)
+        sections = []
+        sections << <<~EO_SECTION if @role && !@role.empty?
+          # Role
+
+          #{Utils::Markdown.align_markdown_headers(@role, level: 2).strip}
+        EO_SECTION
+        sections << <<~EO_SECTION if @objective && !@objective.empty?
+          # Objective
+
+          #{Utils::Markdown.align_markdown_headers(@objective, level: 2).strip}
+        EO_SECTION
+        sections << <<~EO_SECTION if rendered_instructions && !rendered_instructions.empty?
+          # Instructions
+
+          #{Utils::Markdown.align_markdown_headers(rendered_instructions, level: 2).strip}
+        EO_SECTION
+        sections << <<~EO_SECTION if @constraints && !@constraints.empty?
+          # Constraints
+
+          #{Utils::Markdown.align_markdown_headers(@constraints, level: 2).strip}
+        EO_SECTION
+        sections.map(&:strip).join("\n\n")
+      end
+
+      # Render the user prompt
+      # The following instance variables are accessible to render the prompt:
+      # - `@role`
+      # - `@objective`
+      # - `@constraints`
+      #
+      # @param rendered_instructions [String, nil] The rendered instructions, or nil if none
+      # @param input_artifacts [Hash{Symbol => Object}] The input artifacts content for which we render this prompt, per artifact name
+      # @return [String] The rendered user prompt
+      def render_user_prompt(rendered_instructions, input_artifacts:)
+        rendered_instructions || ''
+      end
+
+      # Get the artifact reference name communicated to the assistant
+      #
+      # @param artifact_name [Symbol] The artifact name
+      # @return [String] The artifact reference name used for the assistant
+      def artifact_ref(artifact_name)
+        artifact_name.to_s
+      end
+
+      # Get user instructions for missing output artifacts
+      #
+      # @param missing_output_artifacts [Hash{Symbol => Object}] The missing output artifacts information, per artifact name
+      #   Information can contain the following attributes:
+      #   - description [String] The artifact's description.
+      #   - error [String, nil] An error message related to this missing artifact.
+      # @return [Object] The user instructions (see Instructions#initialize)
+      def missing_output_user_instructions(missing_output_artifacts)
+        <<~EO_PROMPT
+          Some artifacts are missing:
+          #{
+            (
+              missing_output_artifacts.map do |artifact_name, missing_info|
+                "- You must create an artifact named `#{artifact_name}`: #{missing_info[:description]}"
+              end
+            ).join("\n")
+          }
+        EO_PROMPT
+      end
+    end
+  end
+end