roast-ai 0.4.2 → 0.4.3

data/lib/roast/tools/coding_agent.rb

@@ -4,6 +4,7 @@ module Roast
   module Tools
     module CodingAgent
       extend self
+      include Roast::Helpers::MetadataAccess
 
       class CodingAgentError < StandardError; end
 
@@ -11,9 +12,10 @@ module Roast
       private_constant :CONFIG_CODING_AGENT_COMMAND
 
       @configured_command = nil
+      @configured_options = {}
 
       class << self
-        attr_accessor :configured_command
+        attr_accessor :configured_command, :configured_options
 
         def included(base)
           base.class_eval do
@@ -23,11 +25,13 @@ module Roast
               prompt: { type: "string", description: "The prompt to send to Claude Code" },
               include_context_summary: { type: "boolean", description: "Whether to set a summary of the current workflow context as system directive (default: false)", required: false },
               continue: { type: "boolean", description: "Whether to continue where the previous coding agent left off or start with a fresh context (default: false, start fresh)", required: false },
+              retries: { type: "integer", description: "Number of times to retry the coding agent invocation if it terminates with an error (default: 0, no retry)", required: false },
             ) do |params|
               Roast::Tools::CodingAgent.call(
                 params[:prompt],
                 include_context_summary: params[:include_context_summary].presence || false,
                 continue: params[:continue].presence || false,
+                retries: params[:retries],
               )
             end
           end
@@ -36,14 +40,26 @@ module Roast
         # Called after configuration is loaded
         def post_configuration_setup(base, config = {})
           self.configured_command = config[CONFIG_CODING_AGENT_COMMAND]
+          # Store any other configuration options (like model)
+          self.configured_options = config.except(CONFIG_CODING_AGENT_COMMAND)
         end
       end
 
-      def call(prompt, include_context_summary: false, continue: false)
-        Roast::Helpers::Logger.info("🤖 Running CodingAgent\n")
-        run_claude_code(prompt, include_context_summary:, continue:)
+      def call(prompt, include_context_summary: false, continue: false, retries: nil)
+        # Use configured retries as default, fall back to 0 if not configured
+        retries ||= CodingAgent.configured_options[:retries] || CodingAgent.configured_options["retries"] || 0
+        (retries + 1).times do |iteration|
+          Roast::Helpers::Logger.info("🤖 Running CodingAgent#{iteration > 0 ? ", attempt #{iteration + 1} of #{retries + 1}" : ""}\n")
+          return run_claude_code(prompt, include_context_summary:, continue:)
+        rescue CodingAgentError => e
+          raise e if iteration >= retries
+
+          Roast::Helpers::Logger.warn("🤖 Retrying after error running CodingAgent: #{e.message}")
+          Roast::Helpers::Logger.debug(e.backtrace.join("\n") + "\n") if ENV["DEBUG"]
+        end
+        Roast::Helpers::Logger.error("🤖 CodingAgent did not complete successfully after multiple retries")
       rescue StandardError => e
-        "Error running CodingAgent: #{e.message}".tap do |error_message|
+        "🤖 Error running CodingAgent: #{e.message}".tap do |error_message|
           Roast::Helpers::Logger.error(error_message + "\n")
           Roast::Helpers::Logger.debug(e.backtrace.join("\n") + "\n") if ENV["DEBUG"]
         end
@@ -68,9 +84,19 @@ module Roast
           temp_file.write(final_prompt)
           temp_file.close
 
-          # Build the command with continue option if specified
+          # Check for session ID if continue is requested
+          # Resuming from a specific session id is more resilient than simply continuing if there are
+          # parallel invocations of claude being run in the same working directory.
+          session_id = nil
+          if continue
+            session_id = step_metadata["coding_agent_session_id"]
+          end
+
+          # Build the command with continue option (may become resume if session_id exists)
           base_command = claude_code_command
-          command_to_run = build_command(base_command, continue:)
+          command_to_run = build_command(base_command, continue:, session_id:)
+
+          Roast::Helpers::Logger.debug(command_to_run)
 
           # Run Claude Code CLI using the temp file as input with streaming output
           expect_json_output = command_to_run.include?("--output-format stream-json") ||
@@ -85,19 +111,22 @@ module Roast
                 json = parse_json(line)
                 next unless json
 
+                handle_session_info(json)
                 handle_intermediate_message(json)
-                result += handle_result(json) || ""
+                handled_result = handle_result(json)
+                result += handled_result if handled_result
               end
             else
               result = stdout.read
+              # Clear any stale session ID we might have when not using JSON formatting
+              set_current_step_metadata("coding_agent_session_id", nil)
             end
 
             status = wait_thread.value
             if status.success?
               return result
             else
-              error_output = stderr.read
-              return "Error running CodingAgent: #{error_output}"
+              raise CodingAgentError, stderr.read
             end
           end
         ensure
@@ -127,13 +156,18 @@ module Roast
       def handle_result(json)
         if json["type"] == "result"
           # NOTE: the format of an error response is { "subtype": "success", "is_error": true }
-          if json["is_error"]
-            raise CodingAgentError, json["result"]
-          elsif json["subtype"] == "success"
-            json["result"]
-          else
-            raise CodingAgentError, "CodingAgent did not complete successfully: #{line}"
-          end
+          is_error = json["is_error"] || false
+          success = !is_error && json["subtype"] == "success"
+          raise CodingAgentError, json.inspect unless success
+
+          json["result"]
+        end
+      end
+
+      def handle_session_info(json)
+        session_id = json["session_id"]
+        if session_id
+          set_current_step_metadata("coding_agent_session_id", session_id)
         end
       end
 
@@ -150,17 +184,55 @@ module Roast
         CodingAgent.configured_command || ENV["CLAUDE_CODE_COMMAND"] || "claude -p --verbose --output-format stream-json --dangerously-skip-permissions"
       end
 
-      def build_command(base_command, continue:)
-        return base_command unless continue
+      def build_command(base_command, continue:, session_id: nil)
+        command = base_command.dup
 
-        # Add --continue flag to the command
-        # If the command already has flags, insert --continue after 'claude'
-        if base_command.start_with?("claude ")
-          base_command.sub("claude ", "claude --continue ")
-        else
-          # Fallback for non-standard commands
-          "#{base_command} --continue"
+        # Add configured options (like --model), excluding retries which is handled internally
+        options_for_command = CodingAgent.configured_options.except("retries", :retries)
+        if options_for_command.any?
+          options_str = build_options_string(options_for_command)
+          command = if command.start_with?("claude ")
+            command.sub("claude ", "claude #{options_str} ")
+          else
+            # For non-standard commands, append at the end
+            "#{command} #{options_str}"
+          end
         end
+
+        # Add --resume or --continue flag based on continue option and session_id value
+        if continue
+          command = if session_id
+            # Use --resume with session ID if available
+            if command.start_with?("claude ")
+              command.sub("claude ", "claude --resume #{session_id} ")
+            else
+              # Fallback for non-standard commands
+              "#{command} --resume #{session_id}"
+            end
+          elsif command.start_with?("claude ")
+            # Use --continue if no session ID
+            command.sub("claude ", "claude --continue ")
+          else
+            # Fallback for non-standard commands
+            "#{command} --continue"
+          end
+        end
+
+        command
+      end
+
+      def build_options_string(options)
+        options.map do |key, value|
+          # Convert Ruby hash keys to command line format
+          flag = "--#{key.to_s.tr("_", "-")}"
+          if value == true
+            flag
+          elsif value == false || value.nil?
+            nil
+          else
+            "#{flag} #{value}"
+          end
+        end.compact.join(" ")
       end
 
       def prepare_prompt(prompt, include_context_summary)

data/lib/roast/tools/grep.rb

@@ -27,6 +27,10 @@ module Roast
       def call(string)
         Roast::Helpers::Logger.info("🔍 Grepping for string: #{string}\n")
 
+        unless system("command -v rg >/dev/null 2>&1")
+          raise "ripgrep is not available. Please install it using your package manager (e.g., brew install rg) and make sure it's on your PATH."
+        end
+
         # Use Open3 to safely pass the string as an argument, avoiding shell injection
         cmd = ["rg", "-C", "4", "--trim", "--color=never", "--heading", "-F", "--", string, "."]
         stdout, _stderr, _status = Open3.capture3(*cmd)

data/lib/roast/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Roast
-  VERSION = "0.4.2"
+  VERSION = "0.4.3"
 end

data/lib/roast/workflow/agent_step.rb

@@ -3,6 +3,15 @@
 module Roast
   module Workflow
     class AgentStep < BaseStep
+      attr_accessor :include_context_summary, :continue, :resume
+
+      def initialize(workflow, **kwargs)
+        super
+        # Set default values for agent-specific options
+        @include_context_summary = false
+        @continue = false
+      end
+
       def call
         # For inline prompts (detected by plain text step names), use the name as the prompt
         # For file-based steps, load from the prompt file
@@ -12,22 +21,66 @@ module Roast
           read_sidecar_prompt
         end
 
-        # Extract agent-specific configuration from workflow config
-        step_config = workflow.config[name.to_s] || {}
+        # Handle resume option by copying session ID from referenced step
+        if @resume.present?
+          if (session_id = workflow.metadata.dig(@resume, "coding_agent_session_id"))
+            workflow.metadata[name.to_s] ||= {}
+            workflow.metadata[name.to_s]["coding_agent_session_id"] = session_id
+          else
+            Roast::Helpers::Logger.warn("Cannot resume from step '#{@resume}'. It does not have a coding_agent_session_id in its metadata.")
+          end
+        end
+
+        # Use agent-specific configuration that was applied by StepLoader
+        # If resume is set and a session_id is available, translate it to continue
         agent_options = {
-          include_context_summary: step_config.fetch("include_context_summary", false),
-          continue: step_config.fetch("continue", false),
+          include_context_summary: @include_context_summary,
+          continue: @continue || session_id.present?, # Use continue if either continue is set or a session_id to resume from is available
         }
 
         # Call CodingAgent directly with the prompt content and options
         result = Roast::Tools::CodingAgent.call(prompt_content, **agent_options)
 
+        # Parse as JSON if json: true is configured (since CodingAgent response is not handled by Raix)
+        if @json && result.is_a?(String)
+          # Don't try to parse error messages as JSON
+          if result.start_with?("Error running CodingAgent:")
+            raise result
+          end
+
+          # Extract JSON from markdown code blocks anywhere in the response
+          cleaned_result = extract_json_from_markdown(result)
+
+          begin
+            result = JSON.parse(cleaned_result)
+          rescue JSON::ParserError => e
+            raise "Failed to parse CodingAgent result as JSON: #{e.message}"
+          end
+        end
+
         # Process output if print_response is enabled
         process_output(result, print_response:)
 
         # Apply coercion if configured
         apply_coercion(result)
       end
+
+      private
+
+      def extract_json_from_markdown(text)
+        # Look for JSON code blocks anywhere in the text
+        # Matches ```json or ``` followed by content, then closing ```
+        json_block_pattern = /```(?:json)?\s*\n(.*?)\n```/m
+
+        match = text.match(json_block_pattern)
+        if match
+          # Return the content inside the code block
+          match[1].strip
+        else
+          # No code block found, return original text
+          text.strip
+        end
+      end
     end
   end
 end

data/lib/roast/workflow/base_workflow.rb

@@ -24,6 +24,7 @@ module Roast
 
       delegate :api_provider, :openai?, to: :workflow_configuration, allow_nil: true
       delegate :output, :output=, :append_to_final_output, :final_output, to: :output_manager
+      delegate :metadata, :metadata=, to: :metadata_manager
       delegate_missing_to :output
 
       def initialize(file = nil, name: nil, context_path: nil, resource: nil, session_name: nil, workflow_configuration: nil, pre_processing_data: nil)
@@ -38,6 +39,7 @@ module Roast
 
         # Initialize managers
         @output_manager = OutputManager.new
+        @metadata_manager = MetadataManager.new
         @context_manager = ContextManager.new
         @context_management_config = {}
 
@@ -125,8 +127,8 @@ module Roast
         self
       end
 
-      # Expose output manager for state management
-      attr_reader :output_manager
+      # Expose output and metadata managers for state management
+      attr_reader :output_manager, :metadata_manager
 
       private
 

data/lib/roast/workflow/each_step.rb

@@ -54,10 +54,12 @@ module Roast
         # Set the variable in the workflow's context
         workflow.instance_variable_set("@#{@variable_name}", value)
 
-        # Define a getter method for the variable
+        # Define a getter method for the variable if it doesn't exist
         var_name = @variable_name.to_sym
-        workflow.singleton_class.class_eval do
-          attr_reader(var_name)
+        unless workflow.respond_to?(var_name)
+          workflow.singleton_class.class_eval do
+            attr_reader(var_name)
+          end
         end
 
         # Make the variable accessible in the output hash

data/lib/roast/workflow/metadata_manager.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Roast
+  module Workflow
+    # Manages workflow metadata, providing a structure parallel to output
+    # but specifically for internal metadata that shouldn't be user-facing
+    class MetadataManager
+      def initialize
+        @metadata = ActiveSupport::HashWithIndifferentAccess.new
+        @metadata_wrapper = nil
+      end
+
+      # Get metadata wrapped in DotAccessHash for dot notation access
+      def metadata
+        @metadata_wrapper ||= DotAccessHash.new(@metadata)
+      end
+
+      # Set metadata, ensuring it's always a HashWithIndifferentAccess
+      def metadata=(value)
+        @metadata = if value.is_a?(ActiveSupport::HashWithIndifferentAccess)
+          value
+        else
+          ActiveSupport::HashWithIndifferentAccess.new(value)
+        end
+        # Reset the wrapper when metadata changes
+        @metadata_wrapper = nil
+      end
+
+      # Get the raw metadata hash (for internal use)
+      def raw_metadata
+        @metadata
+      end
+
+      # Get a snapshot of the current state for persistence
+      def to_h
+        @metadata.to_h
+      end
+
+      # Restore state from a hash
+      def from_h(data)
+        return unless data
+
+        self.metadata = data
+      end
+    end
+  end
+end

data/lib/roast/workflow/output_handler.rb

@@ -25,6 +25,7 @@ module Roast
           File.write(workflow.output_file, workflow.final_output)
           $stdout.puts "Results saved to #{workflow.output_file}"
         else
+          $stderr.puts "🔥🔥🔥 Final Output: 🔥🔥🔥"
           $stdout.puts workflow.final_output
         end
       end

data/lib/roast/workflow/replay_handler.rb

@@ -77,6 +77,7 @@ module Roast
         return unless state_data && @workflow
 
         restore_output(state_data)
+        restore_metadata(state_data)
         restore_transcript(state_data)
         restore_final_output(state_data)
       end
@@ -88,6 +89,13 @@ module Roast
         @workflow.output = state_data[:output]
       end
 
+      def restore_metadata(state_data)
+        return unless state_data.key?(:metadata)
+        return unless @workflow.respond_to?(:metadata=)
+
+        @workflow.metadata = state_data[:metadata]
+      end
+
       def restore_transcript(state_data)
         return unless state_data.key?(:transcript)
         return unless @workflow.respond_to?(:transcript)

data/lib/roast/workflow/shell_script_step.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+module Roast
+  module Workflow
+    class ShellScriptStep < BaseStep
+      attr_reader :script_path
+      attr_accessor :exit_on_error, :env
+
+      def initialize(workflow, script_path:, **options)
+        super(workflow, **options)
+        @script_path = script_path
+        @exit_on_error = true  # default to true
+        @env = {}              # custom environment variables
+      end
+
+      def call
+        validate_script!
+
+        stdout, stderr, status = execute_script
+
+        result = if status.success?
+          parse_output(stdout)
+        else
+          handle_script_error(stderr, status.exitstatus)
+        end
+
+        process_output(result, print_response: @print_response)
+        result
+      end
+
+      private
+
+      def validate_script!
+        unless File.exist?(script_path)
+          raise ::CLI::Kit::Abort, "Shell script not found: #{script_path}"
+        end
+
+        unless File.executable?(script_path)
+          raise ::CLI::Kit::Abort, "Shell script is not executable: #{script_path}. Run: chmod +x #{script_path}"
+        end
+      end
+
+      def execute_script
+        env = setup_environment
+        cmd = build_command
+
+        log_debug("Executing shell script: #{cmd}")
+        log_debug("Environment: #{env.inspect}")
+
+        Open3.capture3(env, cmd, chdir: Dir.pwd)
+      end
+
+      def build_command
+        script_path
+      end
+
+      def setup_environment
+        env_vars = {}
+
+        # Add workflow context as environment variables
+        env_vars["ROAST_WORKFLOW_RESOURCE"] = workflow.resource.to_s if workflow.resource
+        env_vars["ROAST_STEP_NAME"] = name.value
+
+        # Add workflow outputs as JSON
+        if workflow.output && !workflow.output.empty?
+          env_vars["ROAST_WORKFLOW_OUTPUT"] = JSON.generate(workflow.output)
+        end
+
+        # Add any custom environment variables from step configuration
+        if @env.is_a?(Hash)
+          @env.each do |key, value|
+            env_vars[key.to_s] = value.to_s
+          end
+        end
+
+        env_vars
+      end
+
+      def parse_output(stdout)
+        return "" if stdout.strip.empty?
+
+        if @json
+          begin
+            JSON.parse(stdout.strip)
+          rescue JSON::ParserError => e
+            raise "Failed to parse shell script output as JSON: #{e.message}\nOutput was: #{stdout.strip}"
+          end
+        else
+          stdout.strip
+        end
+      end
+
+      def handle_script_error(stderr, exit_code)
+        error_message = "Shell script failed with exit code #{exit_code}"
+        error_message += "\nError output:\n#{stderr}" unless stderr.strip.empty?
+
+        if @exit_on_error == false
+          log_error(error_message)
+          # Return stderr as the result when not exiting on error
+          stderr.strip.empty? ? "" : stderr.strip
+        else
+          raise ::CLI::Kit::Abort, error_message
+        end
+      end
+
+      def log_debug(message)
+        $stderr.puts "[ShellScriptStep] #{message}" if ENV["ROAST_DEBUG"]
+      end
+
+      def log_error(message)
+        $stderr.puts "[ShellScriptStep] ERROR: #{message}"
+      end
+    end
+  end
+end

data/lib/roast/workflow/state_manager.rb

@@ -42,6 +42,7 @@ module Roast
           order: determine_step_order(step_name),
           transcript: extract_transcript,
           output: extract_output,
+          metadata: extract_metadata,
           final_output: extract_final_output,
           execution_order: extract_execution_order,
         }
@@ -68,6 +69,13 @@ module Roast
         workflow.output.clone
       end
 
+      # Extract metadata if available
+      def extract_metadata
+        return {} unless workflow.respond_to?(:metadata)
+
+        workflow.metadata.clone
+      end
+
       # Extract final output data if available
       def extract_final_output
         return [] unless workflow.respond_to?(:final_output)

data/lib/roast/workflow/step_executor_coordinator.rb

@@ -38,7 +38,7 @@ module Roast
               Kernel.binding.irb # rubocop:disable Lint/Debugger
             end
           else
-            step_orchestrator.execute_step(step, is_last_step: is_last_step)
+            execute_custom_step(step, is_last_step: is_last_step)
           end
         end
       end
@@ -54,6 +54,10 @@ module Roast
       # @return [Object] The result of the step execution
       def execute(step, options = {})
         step_type = StepTypeResolver.resolve(step, @context)
+        step_name = StepTypeResolver.extract_name(step)
+
+        Thread.current[:current_step_name] = step_name if step_name
+        Thread.current[:workflow_metadata] = @context.workflow.metadata
 
         case step_type
         when StepTypeResolver::COMMAND_STEP
@@ -126,14 +130,18 @@ module Roast
         )
       end
 
-      def step_orchestrator
-        dependencies[:step_orchestrator]
-      end
-
       def error_handler
         dependencies[:error_handler]
       end
 
+      def step_loader
+        dependencies[:step_loader]
+      end
+
+      def state_manager
+        dependencies[:state_manager]
+      end
+
       def execute_command_step(step, options)
         exit_on_error = options.fetch(:exit_on_error, true)
         resource_type = @context.resource_type
@@ -185,8 +193,11 @@ module Roast
         step_name = StepTypeResolver.extract_name(step)
 
         # Load and execute the agent step
-        exit_on_error = options.fetch(:exit_on_error, context.exit_on_error?(step))
-        step_orchestrator.execute_step(step_name, exit_on_error:, step_key: options[:step_key], agent_type: :coding_agent)
+        merged_options = options.merge(
+          exit_on_error: options.fetch(:exit_on_error) { context.exit_on_error?(step) },
+          agent_type: :coding_agent,
+        )
+        execute_custom_step(step_name, **merged_options)
       end
 
       def execute_glob_step(step, options = {})
@@ -259,7 +270,7 @@ module Roast
         exit_on_error = options.fetch(:exit_on_error, true)
         step_key = options[:step_key]
         is_last_step = options[:is_last_step]
-        step_orchestrator.execute_step(step, exit_on_error:, step_key:, is_last_step:)
+        execute_custom_step(step, exit_on_error:, step_key:, is_last_step:)
       end
 
       def validate_each_step!(step)
@@ -268,6 +279,30 @@ module Roast
             "Invalid 'each' step format. 'as' and 'steps' must be at the same level as 'each'"
         end
       end
+
+      def execute_custom_step(name, step_key: nil, **options)
+        resource_type = @context.workflow.respond_to?(:resource) ? @context.workflow.resource&.type : nil
+
+        error_handler.with_error_handling(name, resource_type: resource_type) do
+          $stderr.puts "Executing: #{name} (Resource type: #{resource_type || "unknown"})"
+
+          # Use step_key for loading if provided, otherwise use name
+          load_key = step_key || name
+          is_last_step = options[:is_last_step]
+          step_object = step_loader.load(name, exit_on_error: false, step_key: load_key, is_last_step:, **options)
+          step_result = step_object.call
+
+          # Store result in workflow output
+          # Use step_key for output storage if provided (for hash steps)
+          output_key = step_key || name
+          @context.workflow.output[output_key] = step_result
+
+          # Save state after each step
+          state_manager.save_state(name, step_result)
+
+          step_result
+        end
+      end
     end
   end
 end

data/lib/roast/workflow/step_executor_with_reporting.rb

@@ -11,12 +11,12 @@ module Roast
         @name_extractor = StepNameExtractor.new
       end
 
-      def execute(step, options = {})
+      def execute(step, **options)
         # Track tokens before execution
         tokens_before = @context.workflow.context_manager&.total_tokens || 0
 
         # Execute the step
-        result = @base_executor.execute(step, options)
+        result = @base_executor.execute(step, **options)
 
         # Report token consumption after successful execution
         tokens_after = @context.workflow.context_manager&.total_tokens || 0