roast-ai 0.3.1 → 0.4.1

data/lib/roast/workflow/base_step.rb

@@ -1,37 +1,31 @@
 # frozen_string_literal: true
 
-require "erb"
-require "forwardable"
-require "roast/workflow/context_path_resolver"
-
 module Roast
   module Workflow
     class BaseStep
-      extend Forwardable
-
-      attr_accessor :model, :print_response, :json, :params, :resource, :coerce_to
+      attr_accessor :model, :print_response, :json, :params, :resource, :coerce_to, :available_tools
       attr_reader :workflow, :name, :context_path
 
-      def_delegator :workflow, :append_to_final_output
-      def_delegator :workflow, :chat_completion
-      def_delegator :workflow, :transcript
+      delegate :append_to_final_output, :transcript, to: :workflow
+      delegate_missing_to :workflow
 
       # TODO: is this really the model we want to default to, and is this the right place to set it?
       def initialize(workflow, model: "anthropic:claude-opus-4", name: nil, context_path: nil)
         @workflow = workflow
         @model = model
-        @name = name || self.class.name.underscore.split("/").last
+        @name = normalize_name(name)
         @context_path = context_path || ContextPathResolver.resolve(self.class)
         @print_response = false
         @json = false
         @params = {}
         @coerce_to = nil
+        @available_tools = nil
         @resource = workflow.resource if workflow.respond_to?(:resource)
       end
 
       def call
         prompt(read_sidecar_prompt)
-        result = chat_completion(print_response:, json:, params:)
+        result = chat_completion(print_response:, json:, params:, available_tools:)
 
         # Apply coercion if configured
         apply_coercion(result)
@@ -39,25 +33,17 @@ module Roast
 
       protected
 
-      def chat_completion(print_response: nil, json: nil, params: nil)
+      def chat_completion(print_response: nil, json: nil, params: nil, available_tools: nil)
         # Use instance variables as defaults if parameters are not provided
         print_response = @print_response if print_response.nil?
         json = @json if json.nil?
         params = @params if params.nil?
+        available_tools = @available_tools if available_tools.nil?
 
-        workflow.chat_completion(openai: workflow.openai? && model, model: model, json:, params:).tap do |result|
-          process_output(result, print_response:)
-
-          begin
-            if json
-              return nil if result.strip.empty? # Explicitly handle empty string
+        result = workflow.chat_completion(openai: workflow.openai? && model, model: model, json:, params:, available_tools:)
+        process_output(result, print_response:)
 
-              return JSON.parse(result)
-            end
-          rescue JSON::ParserError
-            # If JSON parsing fails, leave it as a string
-          end
-        end
+        result
       end
 
       def prompt(text)
@@ -79,8 +65,20 @@ module Roast
       def process_output(response, print_response:)
         output_path = File.join(context_path, "output.txt")
         if File.exist?(output_path) && print_response
-          # TODO: use the workflow binding or the step?
-          append_to_final_output(ERB.new(File.read(output_path), trim_mode: "-").result(binding))
+          # Deep wrap the response for template access
+          template_response = deep_wrap_for_templates(response)
+
+          # Debug output
+          if template_response.is_a?(DotAccessHash) && template_response.recommendations&.is_a?(Array)
+            $stderr.puts "DEBUG: recommendations array has #{template_response.recommendations.size} items"
+            $stderr.puts "DEBUG: first item class: #{template_response.recommendations.first.class}" if template_response.recommendations.first
+          end
+
+          # Create a binding that includes the wrapped response
+          template_binding = binding
+          template_binding.local_variable_set(:response, template_response)
+
+          append_to_final_output(ERB.new(File.read(output_path), trim_mode: "-").result(template_binding))
         elsif print_response
           append_to_final_output(response)
         end
@@ -88,6 +86,35 @@ module Roast
 
       private
 
+      def normalize_name(name)
+        return name if name.is_a?(Roast::ValueObjects::StepName)
+
+        name_value = name || self.class.name.underscore.split("/").last
+        Roast::ValueObjects::StepName.new(name_value)
+      end
+
+      # Deep wrap response for ERB templates
+      # This creates a new structure where:
+      # - Hashes are wrapped in DotAccessHash
+      # - Arrays are cloned with their Hash elements wrapped
+      def deep_wrap_for_templates(obj)
+        case obj
+        when Hash
+          # Convert the hash to a new hash with wrapped values
+          wrapped_hash = {}
+          obj.each do |key, value|
+            wrapped_hash[key] = deep_wrap_for_templates(value)
+          end
+          DotAccessHash.new(wrapped_hash)
+        when Array
+          # Create a new array with wrapped elements
+          # This allows the template to use dot notation on array elements
+          obj.map { |item| deep_wrap_for_templates(item) }
+        else
+          obj
+        end
+      end
+
       def apply_coercion(result)
         case @coerce_to
         when :boolean
@@ -97,7 +124,6 @@ module Roast
           !!result
         when :llm_boolean
           # Use LLM boolean coercer for natural language responses
-          require "roast/workflow/llm_boolean_coercer"
           LlmBooleanCoercer.coerce(result)
         when :iterable
           # Ensure result is iterable

data/lib/roast/workflow/base_workflow.rb

@@ -1,12 +1,5 @@
 # frozen_string_literal: true
 
-require "raix/chat_completion"
-require "raix/function_dispatch"
-
-require "roast/workflow/context_path_resolver"
-require "roast/workflow/dot_access_hash"
-require "roast/workflow/output_manager"
-
 module Roast
   module Workflow
     class BaseWorkflow
@@ -23,12 +16,15 @@ module Roast
         :session_name,
         :session_timestamp,
         :model,
-        :workflow_configuration
+        :workflow_configuration,
+        :storage_type,
+        :context_management_config
 
-      attr_reader :pre_processing_data
+      attr_reader :pre_processing_data, :context_manager
 
       delegate :api_provider, :openai?, to: :workflow_configuration, allow_nil: true
       delegate :output, :output=, :append_to_final_output, :final_output, to: :output_manager
+      delegate_missing_to :output
 
       def initialize(file = nil, name: nil, context_path: nil, resource: nil, session_name: nil, workflow_configuration: nil, pre_processing_data: nil)
         @file = file
@@ -42,6 +38,8 @@ module Roast
 
         # Initialize managers
         @output_manager = OutputManager.new
+        @context_manager = ContextManager.new
+        @context_management_config = {}
 
         # Setup prompt and handlers
         read_sidecar_prompt.then do |prompt|
@@ -59,29 +57,55 @@ module Roast
         step_model = kwargs[:model]
 
         with_model(step_model) do
+          # Configure context manager if needed
+          if @context_management_config.any?
+            @context_manager.configure(@context_management_config)
+          end
+
+          # Track token usage before API call
+          messages = kwargs[:messages] || transcript.flatten.compact
+          if @context_management_config[:enabled]
+            @context_manager.track_usage(messages)
+            @context_manager.check_warnings
+          end
+
           ActiveSupport::Notifications.instrument("roast.chat_completion.start", {
             model: model,
             parameters: kwargs.except(:openai, :model),
           })
 
+          # Clear any previous response
+          Thread.current[:chat_completion_response] = nil
+
           # Call the parent module's chat_completion
           # skip model because it is read directly from the model method
           result = super(**kwargs.except(:model))
           execution_time = Time.now - start_time
 
+          # Extract token usage from the raw response stored by Raix
+          raw_response = Thread.current[:chat_completion_response]
+          token_usage = extract_token_usage(raw_response) if raw_response
+
+          # Update context manager with actual token usage if available
+          if token_usage && @context_management_config[:enabled]
+            actual_total = token_usage.dig("total_tokens") || token_usage.dig(:total_tokens)
+            @context_manager.update_with_actual_usage(actual_total) if actual_total
+          end
+
           ActiveSupport::Notifications.instrument("roast.chat_completion.complete", {
             success: true,
             model: model,
             parameters: kwargs.except(:openai, :model),
             execution_time: execution_time,
             response_size: result.to_s.length,
+            token_usage: token_usage,
           })
           result
         end
       rescue Faraday::ResourceNotFound => e
         execution_time = Time.now - start_time
         message = e.response.dig(:body, "error", "message") || e.message
-        error = Roast::ResourceNotFoundError.new(message)
+        error = Roast::Errors::ResourceNotFoundError.new(message)
         error.set_backtrace(e.backtrace)
         log_and_raise_error(error, message, step_model || model, kwargs, execution_time)
       rescue => e
@@ -104,19 +128,6 @@ module Roast
       # Expose output manager for state management
       attr_reader :output_manager
 
-      # Allow direct access to output values without 'output.' prefix
-      def method_missing(method_name, *args, &block)
-        if output.respond_to?(method_name)
-          output.send(method_name, *args, &block)
-        else
-          super
-        end
-      end
-
-      def respond_to_missing?(method_name, include_private = false)
-        output.respond_to?(method_name) || super
-      end
-
       private
 
       def log_and_raise_error(error, message, model, params, execution_time)
@@ -134,6 +145,15 @@ module Roast
       def read_sidecar_prompt
         Roast::Helpers::PromptLoader.load_prompt(self, file)
       end
+
+      def extract_token_usage(result)
+        # Token usage is typically in the response metadata
+        # This depends on the API provider's response format
+        return unless result.is_a?(Hash) || result.respond_to?(:to_h)
+
+        result_hash = result.is_a?(Hash) ? result : result.to_h
+        result_hash.dig("usage") || result_hash.dig(:usage)
+      end
     end
   end
 end

data/lib/roast/workflow/case_executor.rb

@@ -24,7 +24,6 @@ module Roast
         raise WorkflowExecutor::ConfigurationError, "Missing 'when' clauses in case configuration" unless when_clauses
 
         # Create and execute a CaseStep
-        require "roast/workflow/case_step" unless defined?(Roast::Workflow::CaseStep)
         case_step = CaseStep.new(
           @workflow,
           config: case_config,

data/lib/roast/workflow/case_step.rb

@@ -1,9 +1,5 @@
 # frozen_string_literal: true
 
-require "roast/workflow/base_step"
-require "roast/workflow/expression_evaluator"
-require "roast/workflow/interpolator"
-
 module Roast
   module Workflow
     class CaseStep < BaseStep

data/lib/roast/workflow/command_executor.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "English"
-
 module Roast
   module Workflow
     class CommandExecutor

data/lib/roast/workflow/conditional_executor.rb

@@ -24,7 +24,6 @@ module Roast
         raise WorkflowExecutor::ConfigurationError, "Missing 'then' steps in conditional configuration" unless then_steps
 
         # Create and execute a ConditionalStep
-        require "roast/workflow/conditional_step" unless defined?(Roast::Workflow::ConditionalStep)
         conditional_step = ConditionalStep.new(
           @workflow,
           config: conditional_config,

data/lib/roast/workflow/conditional_step.rb

@@ -1,9 +1,5 @@
 # frozen_string_literal: true
 
-require "roast/workflow/base_step"
-require "roast/workflow/expression_evaluator"
-require "roast/workflow/interpolator"
-
 module Roast
   module Workflow
     class ConditionalStep < BaseStep

data/lib/roast/workflow/configuration.rb

@@ -1,10 +1,5 @@
 # frozen_string_literal: true
 
-require "roast/workflow/api_configuration"
-require "roast/workflow/configuration_loader"
-require "roast/workflow/resource_resolver"
-require "roast/workflow/step_finder"
-
 module Roast
   module Workflow
     # Encapsulates workflow configuration data and provides structured access
@@ -12,7 +7,7 @@ module Roast
     class Configuration
       MCPTool = Struct.new(:name, :config, :only, :except, keyword_init: true)
 
-      attr_reader :config_hash, :workflow_path, :name, :steps, :pre_processing, :post_processing, :tools, :tool_configs, :mcp_tools, :function_configs, :model, :resource
+      attr_reader :config_hash, :workflow_path, :name, :steps, :pre_processing, :post_processing, :tools, :tool_configs, :mcp_tools, :function_configs, :model, :resource, :context_management
       attr_accessor :target
 
       delegate :api_provider, :openrouter?, :openai?, :uri_base, to: :api_configuration
@@ -37,6 +32,7 @@ module Roast
         @mcp_tools = ConfigurationLoader.extract_mcp_tools(@config_hash)
         @function_configs = ConfigurationLoader.extract_functions(@config_hash)
         @model = ConfigurationLoader.extract_model(@config_hash)
+        @context_management = ConfigurationLoader.extract_context_management(@config_hash)
 
         # Initialize components
         @api_configuration = ApiConfiguration.new(@config_hash)
@@ -45,8 +41,6 @@ module Roast
         # Process target and resource
         @target = ConfigurationLoader.extract_target(@config_hash, options)
         process_resource
-
-        mark_last_step_for_output
       end
 
       def context_path
@@ -98,65 +92,9 @@ module Roast
       attr_reader :api_configuration
 
       def process_resource
-        if defined?(Roast::Resources)
-          @resource = ResourceResolver.resolve(@target, context_path)
-          # Update target with processed value for backward compatibility
-          @target = @resource.value if has_target?
-        end
-      end
-
-      def mark_last_step_for_output
-        return if @steps.empty?
-
-        last_step = find_last_executable_step(@steps.last)
-        return unless last_step
-
-        # Get the step name/key
-        step_key = extract_step_key(last_step)
-        return unless step_key
-
-        # Ensure config exists for this step
-        @config_hash[step_key] ||= {}
-
-        # Only set print_response if not already explicitly configured
-        @config_hash[step_key]["print_response"] = true unless @config_hash[step_key].key?("print_response")
-      end
-
-      def find_last_executable_step(step)
-        case step
-        when String
-          step
-        when Hash
-          # Check if it's a special step type (if, unless, each, repeat, case)
-          if step.key?("if") || step.key?("unless")
-            # For conditional steps, try to find the last step in the "then" branch
-            then_steps = step["then"] || step["steps"]
-            find_last_executable_step(then_steps.last) if then_steps&.any?
-          elsif step.key?("each") || step.key?("repeat")
-            # For iteration steps, we can't reliably determine the last step
-            nil
-          elsif step.key?("case")
-            # For case steps, we can't reliably determine the last step
-            nil
-          elsif step.size == 1
-            # Regular hash step with variable assignment
-            step
-          end
-        when Array
-          # For parallel steps, we can't determine a single "last" step
-          nil
-        else
-          step
-        end
-      end
-
-      def extract_step_key(step)
-        case step
-        when String
-          step
-        when Hash
-          step.keys.first
-        end
+        @resource = ResourceResolver.resolve(@target, context_path)
+        # Update target with processed value for backward compatibility
+        @target = @resource.value if has_target?
       end
     end
   end

data/lib/roast/workflow/configuration_loader.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "yaml"
-
 module Roast
   module Workflow
     # Handles loading and parsing of workflow configuration files
@@ -10,7 +8,7 @@ module Roast
         # Load configuration from a YAML file
         # @param workflow_path [String] Path to the workflow YAML file
         # @return [Hash] The parsed configuration hash
-        def load(workflow_path)
+        def load(workflow_path, options = {})
           validate_path!(workflow_path)
 
           # Load shared.yml if it exists one level above
@@ -25,6 +23,18 @@ module Roast
           end
 
           yaml_content += File.read(workflow_path)
+
+          # Use comprehensive validation if requested
+          if options[:comprehensive_validation]
+            validator = Validators::ValidationOrchestrator.new(yaml_content, workflow_path)
+            unless validator.valid?
+              raise_validation_errors(validator)
+            end
+
+            # Show warnings if any
+            display_warnings(validator.warnings) if validator.warnings.any?
+          end
+
           config_hash = YAML.load(yaml_content, aliases: true)
 
           validate_config!(config_hash)
@@ -134,6 +144,30 @@ module Roast
           options[:target] || config_hash["target"]
         end
 
+        # Extract context management configuration
+        # @param config_hash [Hash] The configuration hash
+        # @return [Hash] The context management configuration with defaults
+        def extract_context_management(config_hash)
+          default_config = {
+            enabled: true,
+            strategy: "auto",
+            threshold: 0.8,
+            max_tokens: nil,
+            retain_steps: [],
+          }
+
+          return default_config unless config_hash["context_management"].is_a?(Hash)
+
+          config = config_hash["context_management"]
+          {
+            enabled: config.fetch("enabled", default_config[:enabled]),
+            strategy: config.fetch("strategy", default_config[:strategy]),
+            threshold: config.fetch("threshold", default_config[:threshold]),
+            max_tokens: config["max_tokens"],
+            retain_steps: config.fetch("retain_steps", default_config[:retain_steps]),
+          }
+        end
+
         private
 
         def validate_path!(workflow_path)
@@ -145,6 +179,32 @@ module Roast
         def validate_config!(config_hash)
           raise ArgumentError, "Invalid workflow configuration" unless config_hash.is_a?(Hash)
         end
+
+        def raise_validation_errors(validator)
+          error_messages = validator.errors.map do |error|
+            message = "• #{error[:message]}"
+            message += " (#{error[:suggestion]})" if error[:suggestion]
+            message
+          end.join("\n")
+
+          raise CLI::Kit::Abort, <<~ERROR
+            Workflow validation failed with #{validator.errors.size} error(s):
+
+            #{error_messages}
+          ERROR
+        end
+
+        def display_warnings(warnings)
+          return if warnings.empty?
+
+          ::CLI::UI::Frame.open("Validation Warnings", color: :yellow) do
+            warnings.each do |warning|
+              puts ::CLI::UI.fmt("{{yellow:#{warning[:message]}}}")
+              puts ::CLI::UI.fmt("  {{gray:→ #{warning[:suggestion]}}}") if warning[:suggestion]
+              puts
+            end
+          end
+        end
       end
     end
   end

data/lib/roast/workflow/configuration_parser.rb

@@ -1,17 +1,11 @@
 # frozen_string_literal: true
 
-require "roast/workflow/configuration"
-require "roast/workflow/workflow_initializer"
-require "roast/workflow/workflow_runner"
-
 module Roast
   module Workflow
     class ConfigurationParser
-      extend Forwardable
-
       attr_reader :configuration, :options, :files, :current_workflow
 
-      def_delegator :current_workflow, :output
+      delegate :output, to: :current_workflow
 
       def initialize(workflow_path, files = [], options = {})
         @configuration = Configuration.new(workflow_path, options)

data/lib/roast/workflow/context_manager.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module Roast
+  module Workflow
+    class ContextManager
+      attr_reader :total_tokens
+
+      def initialize(token_counter: nil, threshold_checker: nil)
+        @token_counter = token_counter || Services::TokenCountingService.new
+        @threshold_checker = threshold_checker || Services::ContextThresholdChecker.new
+        @total_tokens = 0
+        @message_count = 0
+        @config = default_config
+        @last_actual_update = nil
+        @estimated_tokens_since_update = 0
+      end
+
+      def configure(config)
+        @config = default_config.merge(config)
+      end
+
+      def track_usage(messages)
+        current_tokens = @token_counter.count_messages(messages)
+        @total_tokens += current_tokens
+        @message_count += messages.size
+
+        {
+          current_tokens: current_tokens,
+          total_tokens: @total_tokens,
+        }
+      end
+
+      def should_compact?(token_count = @total_tokens)
+        return false unless @config[:enabled]
+
+        @threshold_checker.should_compact?(
+          token_count,
+          @config[:threshold],
+          @config[:max_tokens],
+        )
+      end
+
+      def check_warnings(token_count = @total_tokens)
+        return unless @config[:enabled]
+
+        warning = @threshold_checker.check_warning_threshold(
+          token_count,
+          @config[:threshold],
+          @config[:max_tokens],
+        )
+
+        if warning
+          ActiveSupport::Notifications.instrument("roast.context_warning", warning)
+        end
+      end
+
+      def reset
+        @total_tokens = 0
+        @message_count = 0
+      end
+
+      def statistics
+        {
+          total_tokens: @total_tokens,
+          message_count: @message_count,
+          average_tokens_per_message: @message_count > 0 ? @total_tokens / @message_count : 0,
+        }
+      end
+
+      def update_with_actual_usage(actual_total)
+        return unless actual_total && actual_total > 0
+
+        @total_tokens = actual_total
+        @last_actual_update = Time.now
+        @estimated_tokens_since_update = 0
+      end
+
+      private
+
+      def default_config
+        {
+          enabled: true,
+          threshold: 0.8,
+          max_tokens: nil, # Will use default from threshold checker
+        }
+      end
+    end
+  end
+end

data/lib/roast/workflow/dot_access_hash.rb

@@ -9,7 +9,7 @@ module Roast
 
       def [](key)
         value = @hash[key.to_sym] || @hash[key.to_s]
-        value.is_a?(Hash) ? DotAccessHash.new(value) : value
+        wrap_value(value)
       end
 
       def []=(key, value)
@@ -193,6 +193,21 @@ module Roast
       end
 
       alias_method :member?, :has_key?
+
+      private
+
+      def wrap_value(value)
+        case value
+        when Hash
+          DotAccessHash.new(value)
+        when Array
+          # Don't create a new array - return the original array
+          # Only wrap Hash elements within the array when needed
+          value
+        else
+          value
+        end
+      end
     end
   end
 end

data/lib/roast/workflow/each_step.rb

@@ -65,7 +65,7 @@ module Roast
       end
 
       def save_iteration_state(index, item)
-        state_repository = FileStateRepository.new
+        state_repository = StateRepositoryFactory.create(workflow.storage_type)
 
         # Save the current iteration state
         state_data = {

data/lib/roast/workflow/error_handler.rb

@@ -1,8 +1,5 @@
 # frozen_string_literal: true
 
-require "roast/helpers/logger"
-require "roast/workflow/command_executor"
-
 module Roast
   module Workflow
     # Handles error logging and instrumentation for workflow execution