roast-ai 0.2.1 → 0.2.3

data/lib/roast/workflow/workflow_execution_context.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "active_support/core_ext/string/inflections"
+
+module Roast
+  module Workflow
+    # Manages execution context across pre-processing, target workflows, and post-processing phases
+    class WorkflowExecutionContext
+      attr_reader :pre_processing_output, :target_outputs
+
+      def initialize
+        @pre_processing_output = OutputManager.new
+        @target_outputs = {}
+      end
+
+      # Add output from a target workflow execution
+      def add_target_output(target, output_manager)
+        target_key = generate_target_key(target)
+        @target_outputs[target_key] = output_manager
+      end
+
+      # Get all data as a hash for post-processing
+      def to_h
+        {
+          pre_processing: @pre_processing_output.to_h,
+          targets: @target_outputs.transform_values(&:to_h),
+        }
+      end
+
+      private
+
+      def generate_target_key(target)
+        return "default" unless target
+
+        target.to_s.parameterize
+      end
+    end
+  end
+end

data/lib/roast/workflow/workflow_executor.rb

@@ -48,10 +48,30 @@ module Roast
 
       delegate :workflow, :config_hash, :context_path, to: :context
 
+      # Initialize a new WorkflowExecutor
+      #
+      # @param workflow [BaseWorkflow] The workflow instance to execute
+      # @param config_hash [Hash] The workflow configuration
+      # @param context_path [String] The base path for the workflow
+      # @param error_handler [ErrorHandler] Optional custom error handler
+      # @param step_loader [StepLoader] Optional custom step loader
+      # @param command_executor [CommandExecutor] Optional custom command executor
+      # @param interpolator [Interpolator] Optional custom interpolator
+      # @param state_manager [StateManager] Optional custom state manager
+      # @param iteration_executor [IterationExecutor] Optional custom iteration executor
+      # @param conditional_executor [ConditionalExecutor] Optional custom conditional executor
+      # @param step_orchestrator [StepOrchestrator] Optional custom step orchestrator
+      # @param step_executor_coordinator [StepExecutorCoordinator] Optional custom step executor coordinator
+      # @param phase [Symbol] The execution phase - determines where to load steps from
+      #   Valid values:
+      #   - :steps (default) - Load steps from the main steps directory
+      #   - :pre_processing - Load steps from the pre_processing directory
+      #   - :post_processing - Load steps from the post_processing directory
       def initialize(workflow, config_hash, context_path,
         error_handler: nil, step_loader: nil, command_executor: nil,
         interpolator: nil, state_manager: nil, iteration_executor: nil,
-        conditional_executor: nil, step_orchestrator: nil, step_executor_coordinator: nil)
+        conditional_executor: nil, step_orchestrator: nil, step_executor_coordinator: nil,
+        phase: :steps)
         # Create context object to reduce data clump
         @context = WorkflowContext.new(
           workflow: workflow,
@@ -61,11 +81,11 @@ module Roast
 
         # Dependencies with defaults
         @error_handler = error_handler || ErrorHandler.new
-        @step_loader = step_loader || StepLoader.new(workflow, config_hash, context_path)
+        @step_loader = step_loader || StepLoader.new(workflow, config_hash, context_path, phase: phase)
         @command_executor = command_executor || CommandExecutor.new(logger: @error_handler)
         @interpolator = interpolator || Interpolator.new(workflow, logger: @error_handler)
         @state_manager = state_manager || StateManager.new(workflow, logger: @error_handler)
-        @iteration_executor = iteration_executor || IterationExecutor.new(workflow, context_path, @state_manager)
+        @iteration_executor = iteration_executor || IterationExecutor.new(workflow, context_path, @state_manager, config_hash)
         @conditional_executor = conditional_executor || ConditionalExecutor.new(workflow, context_path, @state_manager, self)
         @step_orchestrator = step_orchestrator || StepOrchestrator.new(workflow, @step_loader, @state_manager, @error_handler, self)
 

data/lib/roast/workflow/workflow_initializer.rb

@@ -81,11 +81,19 @@ module Roast
         end
       end
 
+      def client_options
+        {
+          access_token: @configuration.api_token,
+          uri_base: @configuration.uri_base&.to_s,
+        }.compact
+      end
+
       def configure_openrouter_client
         $stderr.puts "Configuring OpenRouter client with token from workflow"
         require "open_router"
 
-        client = OpenRouter::Client.new(access_token: @configuration.api_token)
+        client = OpenRouter::Client.new(client_options)
+
         Raix.configure do |config|
           config.openrouter_client = client
         end
@@ -96,7 +104,8 @@ module Roast
         $stderr.puts "Configuring OpenAI client with token from workflow"
         require "openai"
 
-        client = OpenAI::Client.new(access_token: @configuration.api_token)
+        client = OpenAI::Client.new(client_options)
+
         Raix.configure do |config|
           config.openai_client = client
         end

data/lib/roast/workflow/workflow_runner.rb

@@ -1,10 +1,12 @@
 # frozen_string_literal: true
 
 require "active_support/notifications"
+require "erb"
 require "roast/workflow/replay_handler"
 require "roast/workflow/workflow_executor"
 require "roast/workflow/output_handler"
 require "roast/workflow/base_workflow"
+require "roast/workflow/dot_access_hash"
 
 module Roast
   module Workflow
@@ -14,6 +16,7 @@ module Roast
         @configuration = configuration
         @options = options
         @output_handler = OutputHandler.new
+        @execution_context = WorkflowExecutionContext.new
       end
 
       def run_for_files(files)
@@ -21,22 +24,65 @@ module Roast
           $stderr.puts "WARNING: Ignoring target parameter because files were provided: #{@configuration.target}"
         end
 
+        # Execute pre-processing steps once before any targets
+        if @configuration.pre_processing.any?
+          $stderr.puts "Running pre-processing steps..."
+          run_pre_processing
+        end
+
+        # Execute main workflow for each file
         files.each do |file|
           $stderr.puts "Running workflow for file: #{file}"
           run_single_workflow(file.strip)
         end
+
+        # Execute post-processing steps once after all targets
+        if @configuration.post_processing.any?
+          $stderr.puts "Running post-processing steps..."
+          run_post_processing
+        end
       end
 
       def run_for_targets
-        @configuration.target.lines.each do |file|
-          $stderr.puts "Running workflow for file: #{file.strip}"
-          run_single_workflow(file.strip)
+        # Split targets by line and clean up
+        target_lines = @configuration.target.lines.map(&:strip).reject(&:empty?)
+
+        # Execute pre-processing steps once before any targets
+        if @configuration.pre_processing.any?
+          $stderr.puts "Running pre-processing steps..."
+          run_pre_processing
+        end
+
+        # Execute main workflow for each target
+        target_lines.each do |file|
+          $stderr.puts "Running workflow for file: #{file}"
+          run_single_workflow(file)
+        end
+
+        # Execute post-processing steps once after all targets
+        if @configuration.post_processing.any?
+          $stderr.puts "Running post-processing steps..."
+          run_post_processing
         end
       end
 
       def run_targetless
         $stderr.puts "Running targetless workflow"
+
+        # Execute pre-processing steps
+        if @configuration.pre_processing.any?
+          $stderr.puts "Running pre-processing steps..."
+          run_pre_processing
+        end
+
+        # Execute main workflow
         run_single_workflow(nil)
+
+        # Execute post-processing steps
+        if @configuration.post_processing.any?
+          $stderr.puts "Running post-processing steps..."
+          run_post_processing
+        end
       end
 
       # Public for backward compatibility with tests
@@ -63,11 +109,59 @@ module Roast
       private
 
       def run_single_workflow(file)
-        workflow = create_workflow(file)
+        # Pass pre-processing data to target workflows
+        # Flatten the structure to remove the 'output' intermediary
+        pre_processing_data = @execution_context.pre_processing_output.raw_output.merge(
+          final_output: @execution_context.pre_processing_output.final_output,
+        )
+        workflow = create_workflow(file, pre_processing_data: pre_processing_data)
         execute_workflow(workflow)
+
+        # Store workflow output in execution context
+        @execution_context.add_target_output(file, workflow.output_manager)
+      end
+
+      def run_pre_processing
+        # Create a workflow for pre-processing (no specific file target)
+        workflow = create_workflow(nil)
+
+        # Execute pre-processing steps
+        executor = WorkflowExecutor.new(workflow, @configuration.config_hash, @configuration.context_path, phase: :pre_processing)
+        executor.execute_steps(@configuration.pre_processing)
+
+        # Store pre-processing output in execution context
+        @execution_context.pre_processing_output.output = workflow.output_manager.raw_output
+        @execution_context.pre_processing_output.final_output = workflow.output_manager.final_output
+      end
+
+      def run_post_processing
+        # Create a workflow for post-processing with access to all results
+        workflow = create_workflow(nil)
+
+        # Pass execution context data to post-processing workflow
+        # Make pre_processing available as a top-level DotNotationHash with flattened structure
+        pre_processing_data = @execution_context.pre_processing_output.raw_output.merge(
+          final_output: @execution_context.pre_processing_output.final_output,
+        )
+        workflow.instance_variable_set(:@pre_processing, DotAccessHash.new(pre_processing_data))
+        workflow.define_singleton_method(:pre_processing) { @pre_processing }
+
+        # Keep targets in output for now
+        workflow.output[:targets] = @execution_context.target_outputs.transform_values(&:to_h)
+
+        # Execute post-processing steps
+        executor = WorkflowExecutor.new(workflow, @configuration.config_hash, @configuration.context_path, phase: :post_processing)
+        executor.execute_steps(@configuration.post_processing)
+
+        # Apply output.txt template if it exists
+        apply_post_processing_template(workflow)
+
+        # Save post-processing outputs
+        @output_handler.save_final_output(workflow)
+        @output_handler.write_results(workflow)
       end
 
-      def create_workflow(file)
+      def create_workflow(file, pre_processing_data: nil)
         BaseWorkflow.new(
           file,
           name: @configuration.basename,
@@ -75,6 +169,7 @@ module Roast
           resource: @configuration.resource,
           session_name: @configuration.name,
           configuration: @configuration,
+          pre_processing_data:,
         ).tap do |workflow|
           workflow.output_file = @options[:output] if @options[:output].present?
           workflow.verbose = @options[:verbose] if @options[:verbose].present?
@@ -82,6 +177,33 @@ module Roast
           workflow.pause_step_name = @options[:pause] if @options[:pause].present?
         end
       end
+
+      def apply_post_processing_template(workflow)
+        # Check for output.txt template in post_processing directory
+        template_path = File.join(@configuration.context_path, "post_processing", "output.txt")
+        return unless File.exist?(template_path)
+
+        # Prepare data for template
+        template_data = {
+          pre_processing: DotAccessHash.new(@execution_context.pre_processing_output.to_h),
+          targets: @execution_context.target_outputs.transform_values { |v| DotAccessHash.new(v.to_h) },
+          output: DotAccessHash.new(workflow.output_manager.raw_output),
+          final_output: workflow.final_output,
+        }
+
+        # Create binding for ERB template with access to template data
+        template_binding = binding
+        template_data.each do |key, value|
+          template_binding.local_variable_set(key, value)
+        end
+
+        # Apply template and append to final output
+        template_content = File.read(template_path)
+        rendered_output = ERB.new(template_content, trim_mode: "-").result(template_binding)
+        workflow.append_to_final_output(rendered_output)
+      rescue => e
+        $stderr.puts "Warning: Failed to apply post-processing output template: #{e.message}"
+      end
     end
   end
 end

data/lib/roast/workflow.rb

@@ -7,6 +7,7 @@ require "roast/workflow/repeat_step"
 require "roast/workflow/each_step"
 require "roast/workflow/base_workflow"
 require "roast/workflow/configuration"
+require "roast/workflow/workflow_execution_context"
 require "roast/workflow/workflow_executor"
 require "roast/workflow/configuration_parser"
 require "roast/workflow/validator"

data/lib/roast.rb

@@ -28,7 +28,13 @@ module Roast
       raise Thor::Error, "Workflow configuration file is required" if paths.empty?
 
       workflow_path, *files = paths
-      expanded_workflow_path = File.expand_path(workflow_path)
+
+      expanded_workflow_path = if workflow_path.include?("workflow.yml")
+        File.expand_path(workflow_path)
+      else
+        File.expand_path("roast/#{workflow_path}/workflow.yml")
+      end
+
       raise Thor::Error, "Expected a Roast workflow configuration file, got directory: #{expanded_workflow_path}" if File.directory?(expanded_workflow_path)
 
       Roast::Workflow::ConfigurationParser.new(expanded_workflow_path, files, options.transform_keys(&:to_sym)).begin!

data/roast.gemspec

@@ -36,7 +36,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency("activesupport", "~> 8.0")
+  spec.add_dependency("activesupport", ">= 7.0")
   spec.add_dependency("cli-ui")
   spec.add_dependency("diff-lcs", "~> 1.5")
   spec.add_dependency("faraday-retry")

data/schema/workflow.json

@@ -41,6 +41,13 @@
           ]
         }
       },
+      "pre_processing": {
+        "type": "array",
+        "description": "Steps executed once before any targets are processed",
+        "items": {
+          "$ref": "#/properties/steps/items"
+        }
+      },
       "steps": {
         "type": "array",
         "items": {
@@ -203,6 +210,13 @@
           ]
         }
       },
+      "post_processing": {
+        "type": "array",
+        "description": "Steps executed once after all targets have been processed",
+        "items": {
+          "$ref": "#/properties/steps/items"
+        }
+      },
       "proceed?": {
         "type": "object",
         "properties": {

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: roast-ai
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.3
 platform: ruby
 authors:
 - Shopify
@@ -13,16 +13,16 @@ dependencies:
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '8.0'
+        version: '7.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '8.0'
+        version: '7.0'
 - !ruby/object:Gem::Dependency
   name: cli-ui
   requirement: !ruby/object:Gem::Requirement
@@ -207,12 +207,28 @@ files:
 - examples/openrouter_example/analyze_input/prompt.md
 - examples/openrouter_example/generate_response/prompt.md
 - examples/openrouter_example/workflow.yml
+- examples/pre_post_processing/README.md
+- examples/pre_post_processing/analyze_test_file/prompt.md
+- examples/pre_post_processing/improve_test_coverage/prompt.md
+- examples/pre_post_processing/optimize_test_performance/prompt.md
+- examples/pre_post_processing/post_processing/aggregate_metrics/prompt.md
+- examples/pre_post_processing/post_processing/cleanup_environment/prompt.md
+- examples/pre_post_processing/post_processing/generate_summary_report/prompt.md
+- examples/pre_post_processing/post_processing/output.txt
+- examples/pre_post_processing/pre_processing/gather_baseline_metrics/prompt.md
+- examples/pre_post_processing/pre_processing/setup_test_environment/prompt.md
+- examples/pre_post_processing/validate_changes/prompt.md
+- examples/pre_post_processing/workflow.yml
 - examples/rspec_to_minitest/README.md
 - examples/rspec_to_minitest/analyze_spec/prompt.md
 - examples/rspec_to_minitest/create_minitest/prompt.md
 - examples/rspec_to_minitest/run_and_improve/prompt.md
 - examples/rspec_to_minitest/workflow.md
 - examples/rspec_to_minitest/workflow.yml
+- examples/single_target_prepost/README.md
+- examples/single_target_prepost/post_processing/output.txt
+- examples/single_target_prepost/pre_processing/gather_dependencies/prompt.md
+- examples/single_target_prepost/workflow.yml
 - examples/smart_coercion_defaults/README.md
 - examples/smart_coercion_defaults/workflow.yml
 - examples/step_configuration/README.md
@@ -224,6 +240,8 @@ files:
 - examples/workflow_generator/info_from_roast.rb
 - examples/workflow_generator/workflow.yml
 - exe/roast
+- gemfiles/activesupport7.gemfile
+- gemfiles/activesupport8.gemfile
 - lib/roast.rb
 - lib/roast/errors.rb
 - lib/roast/factories/api_provider_factory.rb
@@ -253,6 +271,7 @@ files:
 - lib/roast/value_objects.rb
 - lib/roast/value_objects/api_token.rb
 - lib/roast/value_objects/step_name.rb
+- lib/roast/value_objects/uri_base.rb
 - lib/roast/value_objects/workflow_path.rb
 - lib/roast/version.rb
 - lib/roast/workflow.rb
@@ -302,6 +321,7 @@ files:
 - lib/roast/workflow/step_type_resolver.rb
 - lib/roast/workflow/validator.rb
 - lib/roast/workflow/workflow_context.rb
+- lib/roast/workflow/workflow_execution_context.rb
 - lib/roast/workflow/workflow_executor.rb
 - lib/roast/workflow/workflow_initializer.rb
 - lib/roast/workflow/workflow_runner.rb