roast-ai 0.2.0 → 0.2.2

data/lib/roast/workflow/workflow_executor.rb

@@ -48,10 +48,30 @@ module Roast
 
       delegate :workflow, :config_hash, :context_path, to: :context
 
+      # Initialize a new WorkflowExecutor
+      #
+      # @param workflow [BaseWorkflow] The workflow instance to execute
+      # @param config_hash [Hash] The workflow configuration
+      # @param context_path [String] The base path for the workflow
+      # @param error_handler [ErrorHandler] Optional custom error handler
+      # @param step_loader [StepLoader] Optional custom step loader
+      # @param command_executor [CommandExecutor] Optional custom command executor
+      # @param interpolator [Interpolator] Optional custom interpolator
+      # @param state_manager [StateManager] Optional custom state manager
+      # @param iteration_executor [IterationExecutor] Optional custom iteration executor
+      # @param conditional_executor [ConditionalExecutor] Optional custom conditional executor
+      # @param step_orchestrator [StepOrchestrator] Optional custom step orchestrator
+      # @param step_executor_coordinator [StepExecutorCoordinator] Optional custom step executor coordinator
+      # @param phase [Symbol] The execution phase - determines where to load steps from
+      #   Valid values:
+      #   - :steps (default) - Load steps from the main steps directory
+      #   - :pre_processing - Load steps from the pre_processing directory
+      #   - :post_processing - Load steps from the post_processing directory
       def initialize(workflow, config_hash, context_path,
         error_handler: nil, step_loader: nil, command_executor: nil,
         interpolator: nil, state_manager: nil, iteration_executor: nil,
-        conditional_executor: nil, step_orchestrator: nil, step_executor_coordinator: nil)
+        conditional_executor: nil, step_orchestrator: nil, step_executor_coordinator: nil,
+        phase: :steps)
         # Create context object to reduce data clump
         @context = WorkflowContext.new(
           workflow: workflow,
@@ -61,7 +81,7 @@ module Roast
 
         # Dependencies with defaults
         @error_handler = error_handler || ErrorHandler.new
-        @step_loader = step_loader || StepLoader.new(workflow, config_hash, context_path)
+        @step_loader = step_loader || StepLoader.new(workflow, config_hash, context_path, phase: phase)
         @command_executor = command_executor || CommandExecutor.new(logger: @error_handler)
         @interpolator = interpolator || Interpolator.new(workflow, logger: @error_handler)
         @state_manager = state_manager || StateManager.new(workflow, logger: @error_handler)

data/lib/roast/workflow/workflow_initializer.rb

@@ -81,11 +81,19 @@ module Roast
         end
       end
 
+      def client_options
+        {
+          access_token: @configuration.api_token,
+          uri_base: @configuration.uri_base&.to_s,
+        }.compact
+      end
+
       def configure_openrouter_client
         $stderr.puts "Configuring OpenRouter client with token from workflow"
         require "open_router"
 
-        client = OpenRouter::Client.new(access_token: @configuration.api_token)
+        client = OpenRouter::Client.new(client_options)
+
         Raix.configure do |config|
           config.openrouter_client = client
         end
@@ -96,7 +104,8 @@ module Roast
         $stderr.puts "Configuring OpenAI client with token from workflow"
         require "openai"
 
-        client = OpenAI::Client.new(access_token: @configuration.api_token)
+        client = OpenAI::Client.new(client_options)
+
         Raix.configure do |config|
           config.openai_client = client
         end

data/lib/roast/workflow/workflow_runner.rb

@@ -1,10 +1,12 @@
 # frozen_string_literal: true
 
 require "active_support/notifications"
+require "erb"
 require "roast/workflow/replay_handler"
 require "roast/workflow/workflow_executor"
 require "roast/workflow/output_handler"
 require "roast/workflow/base_workflow"
+require "roast/workflow/dot_access_hash"
 
 module Roast
   module Workflow
@@ -14,6 +16,7 @@ module Roast
         @configuration = configuration
         @options = options
         @output_handler = OutputHandler.new
+        @execution_context = WorkflowExecutionContext.new
       end
 
       def run_for_files(files)
@@ -21,22 +24,65 @@ module Roast
           $stderr.puts "WARNING: Ignoring target parameter because files were provided: #{@configuration.target}"
         end
 
+        # Execute pre-processing steps once before any targets
+        if @configuration.pre_processing.any?
+          $stderr.puts "Running pre-processing steps..."
+          run_pre_processing
+        end
+
+        # Execute main workflow for each file
         files.each do |file|
           $stderr.puts "Running workflow for file: #{file}"
           run_single_workflow(file.strip)
         end
+
+        # Execute post-processing steps once after all targets
+        if @configuration.post_processing.any?
+          $stderr.puts "Running post-processing steps..."
+          run_post_processing
+        end
       end
 
       def run_for_targets
-        @configuration.target.lines.each do |file|
-          $stderr.puts "Running workflow for file: #{file.strip}"
-          run_single_workflow(file.strip)
+        # Split targets by line and clean up
+        target_lines = @configuration.target.lines.map(&:strip).reject(&:empty?)
+
+        # Execute pre-processing steps once before any targets
+        if @configuration.pre_processing.any?
+          $stderr.puts "Running pre-processing steps..."
+          run_pre_processing
+        end
+
+        # Execute main workflow for each target
+        target_lines.each do |file|
+          $stderr.puts "Running workflow for file: #{file}"
+          run_single_workflow(file)
+        end
+
+        # Execute post-processing steps once after all targets
+        if @configuration.post_processing.any?
+          $stderr.puts "Running post-processing steps..."
+          run_post_processing
         end
       end
 
       def run_targetless
         $stderr.puts "Running targetless workflow"
+
+        # Execute pre-processing steps
+        if @configuration.pre_processing.any?
+          $stderr.puts "Running pre-processing steps..."
+          run_pre_processing
+        end
+
+        # Execute main workflow
         run_single_workflow(nil)
+
+        # Execute post-processing steps
+        if @configuration.post_processing.any?
+          $stderr.puts "Running post-processing steps..."
+          run_post_processing
+        end
       end
 
       # Public for backward compatibility with tests
@@ -63,11 +109,59 @@ module Roast
       private
 
       def run_single_workflow(file)
-        workflow = create_workflow(file)
+        # Pass pre-processing data to target workflows
+        # Flatten the structure to remove the 'output' intermediary
+        pre_processing_data = @execution_context.pre_processing_output.raw_output.merge(
+          final_output: @execution_context.pre_processing_output.final_output,
+        )
+        workflow = create_workflow(file, pre_processing_data: pre_processing_data)
         execute_workflow(workflow)
+
+        # Store workflow output in execution context
+        @execution_context.add_target_output(file, workflow.output_manager)
+      end
+
+      def run_pre_processing
+        # Create a workflow for pre-processing (no specific file target)
+        workflow = create_workflow(nil)
+
+        # Execute pre-processing steps
+        executor = WorkflowExecutor.new(workflow, @configuration.config_hash, @configuration.context_path, phase: :pre_processing)
+        executor.execute_steps(@configuration.pre_processing)
+
+        # Store pre-processing output in execution context
+        @execution_context.pre_processing_output.output = workflow.output_manager.raw_output
+        @execution_context.pre_processing_output.final_output = workflow.output_manager.final_output
+      end
+
+      def run_post_processing
+        # Create a workflow for post-processing with access to all results
+        workflow = create_workflow(nil)
+
+        # Pass execution context data to post-processing workflow
+        # Make pre_processing available as a top-level DotNotationHash with flattened structure
+        pre_processing_data = @execution_context.pre_processing_output.raw_output.merge(
+          final_output: @execution_context.pre_processing_output.final_output,
+        )
+        workflow.instance_variable_set(:@pre_processing, DotAccessHash.new(pre_processing_data))
+        workflow.define_singleton_method(:pre_processing) { @pre_processing }
+
+        # Keep targets in output for now
+        workflow.output[:targets] = @execution_context.target_outputs.transform_values(&:to_h)
+
+        # Execute post-processing steps
+        executor = WorkflowExecutor.new(workflow, @configuration.config_hash, @configuration.context_path, phase: :post_processing)
+        executor.execute_steps(@configuration.post_processing)
+
+        # Apply output.txt template if it exists
+        apply_post_processing_template(workflow)
+
+        # Save post-processing outputs
+        @output_handler.save_final_output(workflow)
+        @output_handler.write_results(workflow)
       end
 
-      def create_workflow(file)
+      def create_workflow(file, pre_processing_data: nil)
         BaseWorkflow.new(
           file,
           name: @configuration.basename,
@@ -75,6 +169,7 @@ module Roast
           resource: @configuration.resource,
           session_name: @configuration.name,
           configuration: @configuration,
+          pre_processing_data:,
         ).tap do |workflow|
           workflow.output_file = @options[:output] if @options[:output].present?
           workflow.verbose = @options[:verbose] if @options[:verbose].present?
@@ -82,6 +177,33 @@ module Roast
           workflow.pause_step_name = @options[:pause] if @options[:pause].present?
         end
       end
+
+      def apply_post_processing_template(workflow)
+        # Check for output.txt template in post_processing directory
+        template_path = File.join(@configuration.context_path, "post_processing", "output.txt")
+        return unless File.exist?(template_path)
+
+        # Prepare data for template
+        template_data = {
+          pre_processing: DotAccessHash.new(@execution_context.pre_processing_output.to_h),
+          targets: @execution_context.target_outputs.transform_values { |v| DotAccessHash.new(v.to_h) },
+          output: DotAccessHash.new(workflow.output_manager.raw_output),
+          final_output: workflow.final_output,
+        }
+
+        # Create binding for ERB template with access to template data
+        template_binding = binding
+        template_data.each do |key, value|
+          template_binding.local_variable_set(key, value)
+        end
+
+        # Apply template and append to final output
+        template_content = File.read(template_path)
+        rendered_output = ERB.new(template_content, trim_mode: "-").result(template_binding)
+        workflow.append_to_final_output(rendered_output)
+      rescue => e
+        $stderr.puts "Warning: Failed to apply post-processing output template: #{e.message}"
+      end
     end
   end
 end

data/lib/roast/workflow.rb

@@ -7,6 +7,7 @@ require "roast/workflow/repeat_step"
 require "roast/workflow/each_step"
 require "roast/workflow/base_workflow"
 require "roast/workflow/configuration"
+require "roast/workflow/workflow_execution_context"
 require "roast/workflow/workflow_executor"
 require "roast/workflow/configuration_parser"
 require "roast/workflow/validator"

data/lib/roast.rb

@@ -28,7 +28,13 @@ module Roast
       raise Thor::Error, "Workflow configuration file is required" if paths.empty?
 
       workflow_path, *files = paths
-      expanded_workflow_path = File.expand_path(workflow_path)
+
+      expanded_workflow_path = if workflow_path.include?("workflow.yml")
+        File.expand_path(workflow_path)
+      else
+        File.expand_path("roast/#{workflow_path}/workflow.yml")
+      end
+
       raise Thor::Error, "Expected a Roast workflow configuration file, got directory: #{expanded_workflow_path}" if File.directory?(expanded_workflow_path)
 
       Roast::Workflow::ConfigurationParser.new(expanded_workflow_path, files, options.transform_keys(&:to_sym)).begin!

data/roast.gemspec

@@ -36,7 +36,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency("activesupport", "~> 8.0")
+  spec.add_dependency("activesupport", ">= 7.0")
   spec.add_dependency("cli-ui")
   spec.add_dependency("diff-lcs", "~> 1.5")
   spec.add_dependency("faraday-retry")

data/schema/workflow.json

@@ -41,6 +41,13 @@
           ]
         }
       },
+      "pre_processing": {
+        "type": "array",
+        "description": "Steps executed once before any targets are processed",
+        "items": {
+          "$ref": "#/properties/steps/items"
+        }
+      },
       "steps": {
         "type": "array",
         "items": {
@@ -172,10 +179,44 @@
                 }
               },
               "required": ["unless", "then"]
+            },
+            {
+              "type": "object",
+              "properties": {
+                "case": {
+                  "type": "string",
+                  "description": "Expression to evaluate. Can be a Ruby expression in {{...}}, a bash command in $(...), a step name, or prompt content."
+                },
+                "when": {
+                  "type": "object",
+                  "description": "Map of case values to steps to execute when matched",
+                  "additionalProperties": {
+                    "type": "array",
+                    "items": {
+                      "$ref": "#/properties/steps/items"
+                    }
+                  }
+                },
+                "else": {
+                  "type": "array",
+                  "items": {
+                    "$ref": "#/properties/steps/items"
+                  },
+                  "description": "Steps to execute when no when clauses match"
+                }
+              },
+              "required": ["case", "when"]
             }
           ]
         }
       },
+      "post_processing": {
+        "type": "array",
+        "description": "Steps executed once after all targets have been processed",
+        "items": {
+          "$ref": "#/properties/steps/items"
+        }
+      },
       "proceed?": {
         "type": "object",
         "properties": {

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: roast-ai
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.2
 platform: ruby
 authors:
 - Shopify
@@ -13,16 +13,16 @@ dependencies:
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '8.0'
+        version: '7.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '8.0'
+        version: '7.0'
 - !ruby/object:Gem::Dependency
   name: cli-ui
   requirement: !ruby/object:Gem::Requirement
@@ -124,6 +124,7 @@ files:
 - ".ruby-version"
 - CHANGELOG.md
 - CLAUDE.md
+- CLAUDE_NOTES.md
 - CODE_OF_CONDUCT.md
 - CONTRIBUTING.md
 - Gemfile
@@ -141,10 +142,15 @@ files:
 - examples/api_workflow/prompt.md
 - examples/api_workflow/transform_data/prompt.md
 - examples/api_workflow/workflow.yml
+- examples/case_when/README.md
+- examples/case_when/detect_language/prompt.md
+- examples/case_when/workflow.yml
 - examples/conditional/README.md
 - examples/conditional/check_condition/prompt.md
 - examples/conditional/simple_workflow.yml
 - examples/conditional/workflow.yml
+- examples/direct_coerce_syntax/README.md
+- examples/direct_coerce_syntax/workflow.yml
 - examples/dot_notation/README.md
 - examples/dot_notation/workflow.yml
 - examples/exit_on_error/README.md
@@ -195,16 +201,38 @@ files:
 - examples/iteration/update_fix_count/prompt.md
 - examples/iteration/verify_fix/prompt.md
 - examples/iteration/workflow.yml
+- examples/json_handling/README.md
+- examples/json_handling/workflow.yml
 - examples/openrouter_example/README.md
 - examples/openrouter_example/analyze_input/prompt.md
 - examples/openrouter_example/generate_response/prompt.md
 - examples/openrouter_example/workflow.yml
+- examples/pre_post_processing/README.md
+- examples/pre_post_processing/analyze_test_file/prompt.md
+- examples/pre_post_processing/improve_test_coverage/prompt.md
+- examples/pre_post_processing/optimize_test_performance/prompt.md
+- examples/pre_post_processing/post_processing/aggregate_metrics/prompt.md
+- examples/pre_post_processing/post_processing/cleanup_environment/prompt.md
+- examples/pre_post_processing/post_processing/generate_summary_report/prompt.md
+- examples/pre_post_processing/post_processing/output.txt
+- examples/pre_post_processing/pre_processing/gather_baseline_metrics/prompt.md
+- examples/pre_post_processing/pre_processing/setup_test_environment/prompt.md
+- examples/pre_post_processing/validate_changes/prompt.md
+- examples/pre_post_processing/workflow.yml
 - examples/rspec_to_minitest/README.md
 - examples/rspec_to_minitest/analyze_spec/prompt.md
 - examples/rspec_to_minitest/create_minitest/prompt.md
 - examples/rspec_to_minitest/run_and_improve/prompt.md
 - examples/rspec_to_minitest/workflow.md
 - examples/rspec_to_minitest/workflow.yml
+- examples/single_target_prepost/README.md
+- examples/single_target_prepost/post_processing/output.txt
+- examples/single_target_prepost/pre_processing/gather_dependencies/prompt.md
+- examples/single_target_prepost/workflow.yml
+- examples/smart_coercion_defaults/README.md
+- examples/smart_coercion_defaults/workflow.yml
+- examples/step_configuration/README.md
+- examples/step_configuration/workflow.yml
 - examples/workflow_generator/README.md
 - examples/workflow_generator/analyze_user_request/prompt.md
 - examples/workflow_generator/create_workflow_files/prompt.md
@@ -212,6 +240,8 @@ files:
 - examples/workflow_generator/info_from_roast.rb
 - examples/workflow_generator/workflow.yml
 - exe/roast
+- gemfiles/activesupport7.gemfile
+- gemfiles/activesupport8.gemfile
 - lib/roast.rb
 - lib/roast/errors.rb
 - lib/roast/factories/api_provider_factory.rb
@@ -241,6 +271,7 @@ files:
 - lib/roast/value_objects.rb
 - lib/roast/value_objects/api_token.rb
 - lib/roast/value_objects/step_name.rb
+- lib/roast/value_objects/uri_base.rb
 - lib/roast/value_objects/workflow_path.rb
 - lib/roast/version.rb
 - lib/roast/workflow.rb
@@ -248,6 +279,8 @@ files:
 - lib/roast/workflow/base_iteration_step.rb
 - lib/roast/workflow/base_step.rb
 - lib/roast/workflow/base_workflow.rb
+- lib/roast/workflow/case_executor.rb
+- lib/roast/workflow/case_step.rb
 - lib/roast/workflow/command_executor.rb
 - lib/roast/workflow/conditional_executor.rb
 - lib/roast/workflow/conditional_step.rb
@@ -258,6 +291,7 @@ files:
 - lib/roast/workflow/dot_access_hash.rb
 - lib/roast/workflow/each_step.rb
 - lib/roast/workflow/error_handler.rb
+- lib/roast/workflow/expression_evaluator.rb
 - lib/roast/workflow/expression_utils.rb
 - lib/roast/workflow/file_state_repository.rb
 - lib/roast/workflow/interpolator.rb
@@ -287,6 +321,7 @@ files:
 - lib/roast/workflow/step_type_resolver.rb
 - lib/roast/workflow/validator.rb
 - lib/roast/workflow/workflow_context.rb
+- lib/roast/workflow/workflow_execution_context.rb
 - lib/roast/workflow/workflow_executor.rb
 - lib/roast/workflow/workflow_initializer.rb
 - lib/roast/workflow/workflow_runner.rb