roast-ai 0.1.7 → 0.2.0

data/lib/roast/workflow/workflow_executor.rb

@@ -1,239 +1,120 @@
 # frozen_string_literal: true
 
+require "English"
 require "active_support"
 require "active_support/isolated_execution_state"
 require "active_support/notifications"
+require "roast/workflow/command_executor"
+require "roast/workflow/conditional_executor"
+require "roast/workflow/error_handler"
+require "roast/workflow/interpolator"
+require "roast/workflow/iteration_executor"
+require "roast/workflow/parallel_executor"
+require "roast/workflow/state_manager"
+require "roast/workflow/step_executor_factory"
+require "roast/workflow/step_executor_coordinator"
+require "roast/workflow/step_loader"
+require "roast/workflow/step_orchestrator"
+require "roast/workflow/step_type_resolver"
+require "roast/workflow/workflow_context"
 
 module Roast
   module Workflow
     # Handles the execution of workflow steps, including orchestration and threading
+    #
+    # This class now delegates all step execution to StepExecutorCoordinator,
+    # which handles type resolution and execution for all step types.
+    # The circular dependency between executors and workflow has been broken
+    # by introducing the StepRunner interface.
     class WorkflowExecutor
-      DEFAULT_MODEL = "anthropic:claude-3-7-sonnet"
-
-      attr_reader :workflow, :config_hash, :context_path
-
-      def initialize(workflow, config_hash, context_path)
-        @workflow = workflow
-        @config_hash = config_hash
-        @context_path = context_path
-      end
-
-      def execute_steps(steps)
-        steps.each do |step|
-          case step
-          when Hash
-            execute_hash_step(step)
-          when Array
-            execute_parallel_steps(step)
-          when String
-            execute_string_step(step)
-          else
-            raise "Unknown step type: #{step.inspect}"
-          end
-        end
-      end
-
-      # Interpolates {{expression}} in a string with values from the workflow context
-      def interpolate(text)
-        return text unless text.is_a?(String) && text.include?("{{") && text.include?("}}")
-
-        # Replace all {{expression}} with their evaluated values
-        text.gsub(/\{\{([^}]+)\}\}/) do |match|
-          expression = Regexp.last_match(1).strip
-          begin
-            # Evaluate the expression in the workflow's context
-            workflow.instance_eval(expression).to_s
-          rescue => e
-            # If evaluation fails, provide a more detailed error message but preserve the original expression
-            error_msg = "Error interpolating {{#{expression}}}: #{e.message}. This variable is not defined in the workflow context. Please define it before using it in a step name."
-            $stderr.puts "ERROR: #{error_msg}"
-            match # Return the original match to preserve it in the string
-          end
-        end
-      end
-
-      def execute_step(name)
-        start_time = Time.now
-        # For tests, make sure that we handle this gracefully
-        resource_type = workflow.respond_to?(:resource) ? workflow.resource&.type : nil
-
-        ActiveSupport::Notifications.instrument("roast.step.start", {
-          step_name: name,
-          resource_type: resource_type,
-        })
-
-        $stderr.puts "Executing: #{name} (Resource type: #{resource_type || "unknown"})"
-
-        result = if name.starts_with?("$(")
-          strip_and_execute(name).tap do |output|
-            # Add the command and output to the transcript for reference in following steps
-            workflow.transcript << { user: "I just executed the following command: ```\n#{name}\n```\n\nHere is the output:\n\n```\n#{output}\n```" }
-            workflow.transcript << { assistant: "Noted, thank you." }
-          end
-        elsif name.include?("*") && (!workflow.respond_to?(:resource) || !workflow.resource)
-          # Only use the glob method if we don't have a resource object yet
-          # This is for backward compatibility
-          glob(name)
-        else
-          step_object = find_and_load_step(name)
-          step_result = step_object.call
-          workflow.output[name] = step_result
-
-          # Save state after each step if the workflow supports it
-          save_state(name, step_result) if workflow.respond_to?(:session_name) && workflow.session_name
-
-          step_result
-        end
-
-        execution_time = Time.now - start_time
-
-        ActiveSupport::Notifications.instrument("roast.step.complete", {
-          step_name: name,
-          resource_type: resource_type,
-          success: true,
-          execution_time: execution_time,
-          result_size: result.to_s.length,
-        })
-
-        result
-      rescue => e
-        execution_time = Time.now - start_time
-
-        ActiveSupport::Notifications.instrument("roast.step.error", {
-          step_name: name,
-          resource_type: resource_type,
-          error: e.class.name,
-          message: e.message,
-          execution_time: execution_time,
-        })
-        raise
-      end
-
-      private
-
-      def execute_hash_step(step)
-        # execute a command and store the output in a variable
-        name, command = step.to_a.flatten
-
-        # Interpolate variable name if it contains {{}}
-        interpolated_name = interpolate(name)
-
-        if command.is_a?(Hash)
-          execute_steps([command])
-        else
-          # Interpolate command value
-          interpolated_command = interpolate(command)
-          workflow.output[interpolated_name] = execute_step(interpolated_command)
+      # Define custom exception classes for specific error scenarios
+      class WorkflowExecutorError < StandardError
+        attr_reader :step_name, :original_error
+
+        def initialize(message, step_name: nil, original_error: nil)
+          @step_name = step_name
+          @original_error = original_error
+          super(message)
         end
       end
 
-      def execute_parallel_steps(steps)
-        # run steps in parallel, don't proceed until all are done
-        steps.map do |sub_step|
-          Thread.new { execute_steps([sub_step]) }
-        end.each(&:join)
+      class StepExecutionError < WorkflowExecutorError; end
+      class StepNotFoundError < WorkflowExecutorError; end
+      class InterpolationError < WorkflowExecutorError; end
+      class StateError < WorkflowExecutorError; end
+      class ConfigurationError < WorkflowExecutorError; end
+
+      attr_reader :context, :step_loader, :state_manager, :step_executor_coordinator
+
+      delegate :workflow, :config_hash, :context_path, to: :context
+
+      def initialize(workflow, config_hash, context_path,
+        error_handler: nil, step_loader: nil, command_executor: nil,
+        interpolator: nil, state_manager: nil, iteration_executor: nil,
+        conditional_executor: nil, step_orchestrator: nil, step_executor_coordinator: nil)
+        # Create context object to reduce data clump
+        @context = WorkflowContext.new(
+          workflow: workflow,
+          config_hash: config_hash,
+          context_path: context_path,
+        )
+
+        # Dependencies with defaults
+        @error_handler = error_handler || ErrorHandler.new
+        @step_loader = step_loader || StepLoader.new(workflow, config_hash, context_path)
+        @command_executor = command_executor || CommandExecutor.new(logger: @error_handler)
+        @interpolator = interpolator || Interpolator.new(workflow, logger: @error_handler)
+        @state_manager = state_manager || StateManager.new(workflow, logger: @error_handler)
+        @iteration_executor = iteration_executor || IterationExecutor.new(workflow, context_path, @state_manager)
+        @conditional_executor = conditional_executor || ConditionalExecutor.new(workflow, context_path, @state_manager, self)
+        @step_orchestrator = step_orchestrator || StepOrchestrator.new(workflow, @step_loader, @state_manager, @error_handler, self)
+
+        # Initialize coordinator with dependencies
+        @step_executor_coordinator = step_executor_coordinator || StepExecutorCoordinator.new(
+          context: @context,
+          dependencies: {
+            workflow_executor: self,
+            interpolator: @interpolator,
+            command_executor: @command_executor,
+            iteration_executor: @iteration_executor,
+            conditional_executor: @conditional_executor,
+            step_orchestrator: @step_orchestrator,
+            error_handler: @error_handler,
+          },
+        )
       end
 
-      def execute_string_step(step)
-        # Interpolate any {{}} expressions before executing the step
-        interpolated_step = interpolate(step)
-        execute_step(interpolated_step)
+      # Logger interface methods for backward compatibility
+      def log_error(message)
+        @error_handler.log_error(message)
       end
 
-      def find_and_load_step(step_name)
-        # First check for a prompt step
-        if step_name.strip.include?(" ")
-          return Roast::Workflow::PromptStep.new(workflow, name: step_name, auto_loop: false)
-        end
-
-        # First check for a ruby file with the step name
-        rb_file_path = File.join(context_path, "#{step_name}.rb")
-        if File.file?(rb_file_path)
-          return load_ruby_step(rb_file_path, step_name)
-        end
-
-        # Check in shared directory for ruby file
-        shared_rb_path = File.expand_path(File.join(context_path, "..", "shared", "#{step_name}.rb"))
-        if File.file?(shared_rb_path)
-          return load_ruby_step(shared_rb_path, step_name, File.dirname(shared_rb_path))
-        end
-
-        # Continue with existing directory check logic
-        step_path = File.join(context_path, step_name)
-        step_path = File.expand_path(File.join(context_path, "..", "shared", step_name)) unless File.directory?(step_path)
-        raise "Step directory or file not found: #{step_path}" unless File.directory?(step_path)
-
-        setup_step(Roast::Workflow::BaseStep, step_name, step_path)
+      def log_warning(message)
+        @error_handler.log_warning(message)
       end
 
-      def glob(name)
-        Dir.glob(name).join("\n")
+      def warn(message)
+        @error_handler.log_warning(message)
       end
 
-      def load_ruby_step(file_path, step_name, context_path = File.dirname(file_path))
-        $stderr.puts "Requiring step file: #{file_path}"
-        require file_path
-        step_class = step_name.classify.constantize
-        setup_step(step_class, step_name, context_path)
+      def error(message)
+        @error_handler.log_error(message)
       end
 
-      def setup_step(step_class, step_name, context_path)
-        step_class.new(workflow, name: step_name, context_path: context_path).tap do |step|
-          step_config = config_hash[step_name]
-
-          # Always set the model, even if there's no step_config
-          # Use step-specific model if defined, otherwise use workflow default model, or fallback to DEFAULT_MODEL
-          step.model = step_config&.dig("model") || config_hash["model"] || DEFAULT_MODEL
-
-          # Pass resource to step if supported
-          step.resource = workflow.resource if step.respond_to?(:resource=)
-
-          if step_config.present?
-            step.print_response = step_config["print_response"] if step_config["print_response"].present?
-            step.loop = step_config["loop"] if step_config["loop"].present?
-            step.json = step_config["json"] if step_config["json"].present?
-            step.params = step_config["params"] if step_config["params"].present?
-          end
-        end
+      def execute_steps(workflow_steps)
+        @step_executor_coordinator.execute_steps(workflow_steps)
       end
 
-      def strip_and_execute(step)
-        if step.match?(/^\$\((.*)\)$/)
-          # Extract the command from the $(command) syntax
-          command = step.strip.match(/^\$\((.*)\)$/)[1]
-
-          # NOTE: We don't need to call interpolate here as it's already been done
-          # in execute_string_step before this method is called
-          %x(#{command})
-        else
-          raise "Missing closing parentheses: #{step}"
-        end
+      def interpolate(text)
+        @interpolator.interpolate(text)
       end
 
-      def save_state(step_name, step_result)
-        state_repository = FileStateRepository.new
-
-        # Gather necessary data for state
-        static_data = workflow.respond_to?(:transcript) ? workflow.transcript.map(&:itself) : []
-
-        # Get output and final_output if available
-        output = workflow.respond_to?(:output) ? workflow.output.clone : {}
-        final_output = workflow.respond_to?(:final_output) ? workflow.final_output.clone : []
-
-        state_data = {
-          step_name: step_name,
-          order: output.keys.index(step_name) || output.size,
-          transcript: static_data,
-          output: output,
-          final_output: final_output,
-          execution_order: output.keys,
-        }
-
-        # Save the state
-        state_repository.save_state(workflow, step_name, state_data)
-      rescue => e
-        # Don't fail the workflow if state saving fails
-        $stderr.puts "Warning: Failed to save workflow state: #{e.message}"
+      def execute_step(name, exit_on_error: true)
+        @step_executor_coordinator.execute(name, exit_on_error: exit_on_error)
+      rescue StepLoader::StepNotFoundError => e
+        raise StepNotFoundError.new(e.message, step_name: e.step_name, original_error: e.original_error)
+      rescue StepLoader::StepExecutionError => e
+        raise StepExecutionError.new(e.message, step_name: e.step_name, original_error: e.original_error)
       end
     end
   end

data/lib/roast/workflow/workflow_initializer.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+require "raix"
+require "roast/initializers"
+require "roast/helpers/function_caching_interceptor"
+require "roast/helpers/logger"
+require "roast/workflow/base_workflow"
+
+module Roast
+  module Workflow
+    # Handles initialization of workflow dependencies: initializers, tools, and API clients
+    class WorkflowInitializer
+      def initialize(configuration)
+        @configuration = configuration
+      end
+
+      def setup
+        load_roast_initializers
+        include_tools
+        configure_api_client
+      end
+
+      private
+
+      def load_roast_initializers
+        Roast::Initializers.load_all
+      end
+
+      def include_tools
+        return unless @configuration.tools.present?
+
+        BaseWorkflow.include(Raix::FunctionDispatch)
+        BaseWorkflow.include(Roast::Helpers::FunctionCachingInterceptor) # Add caching support
+        BaseWorkflow.include(*@configuration.tools.map(&:constantize))
+      end
+
+      def configure_api_client
+        # Skip if api client is already configured (e.g., by initializers)
+        return if api_client_already_configured?
+
+        # Skip if no api_token is provided in the workflow
+        return if @configuration.api_token.blank?
+
+        client = case @configuration.api_provider
+        when :openrouter
+          configure_openrouter_client
+        when :openai
+          configure_openai_client
+        when nil
+          # Skip configuration if no api_provider is set
+          return
+        else
+          raise "Unsupported api_provider in workflow configuration: #{@configuration.api_provider}"
+        end
+
+        # Validate the client configuration by making a test API call
+        validate_api_client(client) if client
+      rescue OpenRouter::ConfigurationError, Faraday::UnauthorizedError => e
+        error = Roast::AuthenticationError.new("API authentication failed: No API token provided or token is invalid")
+        error.set_backtrace(e.backtrace)
+
+        ActiveSupport::Notifications.instrument("roast.workflow.start.error", {
+          error: error.class.name,
+          message: error.message,
+        })
+
+        raise error
+      rescue => e
+        Roast::Helpers::Logger.error("Error configuring API client: #{e.message}")
+        raise e
+      end
+
+      def api_client_already_configured?
+        case @configuration.api_provider
+        when :openrouter
+          Raix.configuration.openrouter_client.present?
+        when :openai
+          Raix.configuration.openai_client.present?
+        else
+          false
+        end
+      end
+
+      def configure_openrouter_client
+        $stderr.puts "Configuring OpenRouter client with token from workflow"
+        require "open_router"
+
+        client = OpenRouter::Client.new(access_token: @configuration.api_token)
+        Raix.configure do |config|
+          config.openrouter_client = client
+        end
+        client
+      end
+
+      def configure_openai_client
+        $stderr.puts "Configuring OpenAI client with token from workflow"
+        require "openai"
+
+        client = OpenAI::Client.new(access_token: @configuration.api_token)
+        Raix.configure do |config|
+          config.openai_client = client
+        end
+        client
+      end
+
+      def validate_api_client(client)
+        # Make a lightweight API call to validate the token
+        client.models.list if client.respond_to?(:models)
+      end
+    end
+  end
+end

data/lib/roast/workflow/workflow_runner.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+require "active_support/notifications"
+require "roast/workflow/replay_handler"
+require "roast/workflow/workflow_executor"
+require "roast/workflow/output_handler"
+require "roast/workflow/base_workflow"
+
+module Roast
+  module Workflow
+    # Handles running workflows for files/targets and orchestrating execution
+    class WorkflowRunner
+      def initialize(configuration, options = {})
+        @configuration = configuration
+        @options = options
+        @output_handler = OutputHandler.new
+      end
+
+      def run_for_files(files)
+        if @configuration.has_target?
+          $stderr.puts "WARNING: Ignoring target parameter because files were provided: #{@configuration.target}"
+        end
+
+        files.each do |file|
+          $stderr.puts "Running workflow for file: #{file}"
+          run_single_workflow(file.strip)
+        end
+      end
+
+      def run_for_targets
+        @configuration.target.lines.each do |file|
+          $stderr.puts "Running workflow for file: #{file.strip}"
+          run_single_workflow(file.strip)
+        end
+      end
+
+      def run_targetless
+        $stderr.puts "Running targetless workflow"
+        run_single_workflow(nil)
+      end
+
+      # Public for backward compatibility with tests
+      def execute_workflow(workflow)
+        steps = @configuration.steps
+
+        # Handle replay option
+        if @options[:replay]
+          replay_handler = ReplayHandler.new(workflow)
+          steps = replay_handler.process_replay(steps, @options[:replay])
+        end
+
+        # Execute the steps
+        executor = WorkflowExecutor.new(workflow, @configuration.config_hash, @configuration.context_path)
+        executor.execute_steps(steps)
+
+        $stderr.puts "🔥🔥🔥 ROAST COMPLETE! 🔥🔥🔥"
+
+        # Save outputs
+        @output_handler.save_final_output(workflow)
+        @output_handler.write_results(workflow)
+      end
+
+      private
+
+      def run_single_workflow(file)
+        workflow = create_workflow(file)
+        execute_workflow(workflow)
+      end
+
+      def create_workflow(file)
+        BaseWorkflow.new(
+          file,
+          name: @configuration.basename,
+          context_path: @configuration.context_path,
+          resource: @configuration.resource,
+          session_name: @configuration.name,
+          configuration: @configuration,
+        ).tap do |workflow|
+          workflow.output_file = @options[:output] if @options[:output].present?
+          workflow.verbose = @options[:verbose] if @options[:verbose].present?
+          workflow.concise = @options[:concise] if @options[:concise].present?
+          workflow.pause_step_name = @options[:pause] if @options[:pause].present?
+        end
+      end
+    end
+  end
+end

data/lib/roast/workflow.rb

@@ -2,6 +2,9 @@
 
 require "roast/workflow/base_step"
 require "roast/workflow/prompt_step"
+require "roast/workflow/base_iteration_step"
+require "roast/workflow/repeat_step"
+require "roast/workflow/each_step"
 require "roast/workflow/base_workflow"
 require "roast/workflow/configuration"
 require "roast/workflow/workflow_executor"

data/lib/roast.rb

@@ -1,13 +1,16 @@
 # frozen_string_literal: true
 
+require "fileutils"
+require "cli/ui"
 require "raix"
 require "thor"
-require "roast/version"
-require "roast/tools"
+require "roast/errors"
 require "roast/helpers"
+require "roast/initializers"
 require "roast/resources"
+require "roast/tools"
+require "roast/version"
 require "roast/workflow"
-require "roast/initializers"
 
 module Roast
   ROOT = File.expand_path("../..", __FILE__)
@@ -19,6 +22,8 @@ module Roast
     option :verbose, type: :boolean, aliases: "-v", desc: "Show output from all steps as they are executed"
     option :target, type: :string, aliases: "-t", desc: "Override target files. Can be file path, glob pattern, or $(shell command)"
     option :replay, type: :string, aliases: "-r", desc: "Resume workflow from a specific step. Format: step_name or session_timestamp:step_name"
+    option :pause, type: :string, aliases: "-p", desc: "Pause workflow after a specific step. Format: step_name"
+
     def execute(*paths)
       raise Thor::Error, "Workflow configuration file is required" if paths.empty?
 
@@ -34,6 +39,94 @@ module Roast
       puts "Roast version #{Roast::VERSION}"
     end
 
+    desc "init", "Initialize a new Roast workflow from an example"
+    option :example, type: :string, aliases: "-e", desc: "Name of the example to use directly (skips picker)"
+    def init
+      if options[:example]
+        copy_example(options[:example])
+      else
+        show_example_picker
+      end
+    end
+
+    private
+
+    def show_example_picker
+      examples = available_examples
+
+      if examples.empty?
+        puts "No examples found!"
+        return
+      end
+
+      puts "Select an option:"
+      choices = ["Pick from examples", "New from prompt (beta)"]
+
+      selected = run_picker(choices, "Select initialization method:")
+
+      case selected
+      when "Pick from examples"
+        example_choice = run_picker(examples, "Select an example:")
+        copy_example(example_choice) if example_choice
+      when "New from prompt (beta)"
+        create_from_prompt
+      end
+    end
+
+    def available_examples
+      examples_dir = File.join(Roast::ROOT, "examples")
+      return [] unless File.directory?(examples_dir)
+
+      Dir.entries(examples_dir)
+        .select { |entry| File.directory?(File.join(examples_dir, entry)) && entry != "." && entry != ".." }
+        .sort
+    end
+
+    def run_picker(options, prompt)
+      return if options.empty?
+
+      ::CLI::UI::Prompt.ask(prompt) do |handler|
+        options.each { |option| handler.option(option) { |selection| selection } }
+      end
+    end
+
+    def copy_example(example_name)
+      examples_dir = File.join(Roast::ROOT, "examples")
+      source_path = File.join(examples_dir, example_name)
+      target_path = File.join(Dir.pwd, example_name)
+
+      unless File.directory?(source_path)
+        puts "Example '#{example_name}' not found!"
+        return
+      end
+
+      if File.exist?(target_path)
+        puts "Directory '#{example_name}' already exists in current directory!"
+        return
+      end
+
+      FileUtils.cp_r(source_path, target_path)
+      puts "Successfully copied example '#{example_name}' to current directory."
+    end
+
+    def create_from_prompt
+      puts("Create a new workflow from a description")
+      puts
+
+      # Execute the workflow generator
+      generator_path = File.join(Roast::ROOT, "examples", "workflow_generator", "workflow.yml")
+
+      begin
+        # Execute the workflow generator (it will handle user input)
+        Roast::Workflow::ConfigurationParser.new(generator_path, [], {}).begin!
+
+        puts
+        puts("Workflow generation complete!")
+      rescue => e
+        puts("Error generating workflow: #{e.message}")
+      end
+    end
+
     class << self
       def exit_on_failure?
         true

data/roast.gemspec

@@ -37,9 +37,10 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency("activesupport", "~> 8.0")
+  spec.add_dependency("cli-ui")
   spec.add_dependency("diff-lcs", "~> 1.5")
   spec.add_dependency("faraday-retry")
   spec.add_dependency("json-schema")
-  spec.add_dependency("raix", "~> 0.8.4")
+  spec.add_dependency("raix", "~> 0.8.6")
   spec.add_dependency("thor", "~> 1.3")
 end