roast-ai 0.3.1 → 0.4.1

data/lib/roast/workflow/validators/schema_validator.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module Roast
+  module Workflow
+    module Validators
+      # Validates workflow configuration against JSON schema
+      class SchemaValidator < BaseValidator
+        attr_reader :parsed_yaml
+
+        def initialize(yaml_content, workflow_path = nil) # rubocop:disable Lint/MissingSuper
+          @yaml_content = yaml_content&.strip || ""
+          @workflow_path = workflow_path
+          @errors = []
+          @warnings = []
+
+          begin
+            @parsed_yaml = @yaml_content.empty? ? {} : YAML.safe_load(@yaml_content)
+          rescue Psych::SyntaxError => e
+            @errors << format_yaml_error(e)
+            @parsed_yaml = {}
+          end
+        end
+
+        def validate
+          if @parsed_yaml.empty?
+            @errors << {
+              type: :empty_configuration,
+              message: "Workflow configuration is empty",
+              suggestion: "Provide a valid workflow configuration with required fields: name, tools, and steps",
+            }
+            return
+          end
+
+          validator = Validator.new(@yaml_content)
+          unless validator.valid?
+            validator.errors.each do |error|
+              @errors << format_schema_error(error)
+            end
+          end
+        end
+
+        private
+
+        def format_yaml_error(error)
+          {
+            type: :yaml_syntax,
+            message: "YAML syntax error: #{error.message}",
+            line: error.line,
+            column: error.column,
+            suggestion: "Check YAML syntax at line #{error.line}, column #{error.column}",
+          }
+        end
+
+        def format_schema_error(error)
+          # Parse JSON Schema error and make it more user-friendly
+          if error.include?("did not contain a required property")
+            # Extract property name from error
+            match = error.match(/required property of '([^']+)'/)
+            if match
+              property = match[1]
+              {
+                type: :schema,
+                message: "Missing required field: '#{property}'",
+                suggestion: "Add '#{property}' to your workflow configuration",
+              }
+            else
+              {
+                type: :schema,
+                message: error,
+                suggestion: "Check the required fields in your workflow configuration",
+              }
+            end
+          elsif error.include?("does not match")
+            {
+              type: :schema,
+              message: error,
+              suggestion: "Check the workflow schema documentation for valid values",
+            }
+          else
+            {
+              type: :schema,
+              message: error,
+              suggestion: "Refer to the workflow schema for correct configuration structure",
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/roast/workflow/validators/step_collector.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Roast
+  module Workflow
+    module Validators
+      # Collects and caches all steps from a workflow configuration
+      class StepCollector
+        def initialize(parsed_yaml)
+          @parsed_yaml = parsed_yaml
+          @all_steps = nil
+        end
+
+        def all_steps
+          @all_steps ||= collect_all_steps(@parsed_yaml)
+        end
+
+        private
+
+        def collect_all_steps(config, steps = [])
+          # Recursively collect all steps from the configuration
+          ["steps", "pre_processing", "post_processing"].each do |key|
+            if config[key]
+              steps.concat(extract_steps_from_array(config[key]))
+            end
+          end
+          steps
+        end
+
+        def extract_steps_from_array(steps_array, collected = [])
+          steps_array.each do |step|
+            case step
+            when String
+              collected << step
+            when Hash
+              if step["steps"]
+                collected.concat(extract_steps_from_array(step["steps"]))
+              end
+              # Handle conditional steps
+              ["then", "else", "true", "false"].each do |branch|
+                if step[branch]
+                  collected.concat(extract_steps_from_array(step[branch]))
+                end
+              end
+              # Handle case/when steps
+              step["when"]&.each_value do |when_steps|
+                collected.concat(extract_steps_from_array(when_steps))
+              end
+            when Array
+              collected.concat(extract_steps_from_array(step))
+            end
+          end
+          collected
+        end
+      end
+    end
+  end
+end

data/lib/roast/workflow/validators/validation_orchestrator.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Roast
+  module Workflow
+    module Validators
+      # Orchestrates all validators and aggregates results
+      class ValidationOrchestrator
+        attr_reader :errors, :warnings
+
+        def initialize(yaml_content, workflow_path = nil)
+          @yaml_content = yaml_content
+          @workflow_path = workflow_path
+          @errors = []
+          @warnings = []
+        end
+
+        def valid?
+          # First run schema validation
+          schema_validator = SchemaValidator.new(@yaml_content, @workflow_path)
+
+          unless schema_validator.valid?
+            @errors = schema_validator.errors
+            @warnings = schema_validator.warnings
+            return false
+          end
+
+          parsed_yaml = schema_validator.parsed_yaml
+
+          # If schema is valid, run other validators
+          if @errors.empty?
+            step_collector = StepCollector.new(parsed_yaml)
+
+            # Run dependency validation
+            dependency_validator = DependencyValidator.new(parsed_yaml, @workflow_path, step_collector: step_collector)
+            dependency_validator.validate
+            @errors.concat(dependency_validator.errors)
+            @warnings.concat(dependency_validator.warnings)
+
+            # Run linting only if no errors
+            if @errors.empty?
+              linting_validator = LintingValidator.new(parsed_yaml, @workflow_path, step_collector: step_collector)
+              linting_validator.validate
+              @warnings.concat(linting_validator.warnings)
+            end
+          end
+
+          @errors.empty?
+        end
+      end
+    end
+  end
+end

data/lib/roast/workflow/workflow_executor.rb

@@ -1,21 +1,5 @@
 # frozen_string_literal: true
 
-require "English"
-
-require "roast/workflow/command_executor"
-require "roast/workflow/conditional_executor"
-require "roast/workflow/error_handler"
-require "roast/workflow/interpolator"
-require "roast/workflow/iteration_executor"
-require "roast/workflow/parallel_executor"
-require "roast/workflow/state_manager"
-require "roast/workflow/step_executor_factory"
-require "roast/workflow/step_executor_coordinator"
-require "roast/workflow/step_loader"
-require "roast/workflow/step_orchestrator"
-require "roast/workflow/step_type_resolver"
-require "roast/workflow/workflow_context"
-
 module Roast
   module Workflow
     # Handles the execution of workflow steps, including orchestration and threading
@@ -82,13 +66,13 @@ module Roast
         @step_loader = step_loader || StepLoader.new(workflow, config_hash, context_path, phase: phase)
         @command_executor = command_executor || CommandExecutor.new(logger: @error_handler)
         @interpolator = interpolator || Interpolator.new(workflow, logger: @error_handler)
-        @state_manager = state_manager || StateManager.new(workflow, logger: @error_handler)
+        @state_manager = state_manager || StateManager.new(workflow, logger: @error_handler, storage_type: workflow.storage_type)
         @iteration_executor = iteration_executor || IterationExecutor.new(workflow, context_path, @state_manager, config_hash)
         @conditional_executor = conditional_executor || ConditionalExecutor.new(workflow, context_path, @state_manager, self)
         @step_orchestrator = step_orchestrator || StepOrchestrator.new(workflow, @step_loader, @state_manager, @error_handler, self)
 
         # Initialize coordinator with dependencies
-        @step_executor_coordinator = step_executor_coordinator || StepExecutorCoordinator.new(
+        base_coordinator = step_executor_coordinator || StepExecutorCoordinator.new(
           context: @context,
           dependencies: {
             workflow_executor: self,
@@ -100,6 +84,13 @@ module Roast
             error_handler: @error_handler,
           },
         )
+
+        # Only wrap with reporting decorator if workflow has token tracking enabled
+        @step_executor_coordinator = if workflow.respond_to?(:context_manager) && workflow.context_manager
+          StepExecutorWithReporting.new(base_coordinator, @context)
+        else
+          base_coordinator
+        end
       end
 
       # Logger interface methods for backward compatibility
@@ -127,8 +118,8 @@ module Roast
         @interpolator.interpolate(text)
       end
 
-      def execute_step(name, exit_on_error: true)
-        @step_executor_coordinator.execute(name, exit_on_error: exit_on_error)
+      def execute_step(name, exit_on_error: true, is_last_step: nil)
+        @step_executor_coordinator.execute(name, exit_on_error:, is_last_step:)
       rescue StepLoader::StepNotFoundError => e
         raise StepNotFoundError.new(e.message, step_name: e.step_name, original_error: e.original_error)
       rescue StepLoader::StepExecutionError => e

data/lib/roast/workflow/workflow_initializer.rb

@@ -1,12 +1,5 @@
 # frozen_string_literal: true
 
-require "raix"
-require "roast/initializers"
-require "roast/helpers/function_caching_interceptor"
-require "roast/helpers/logger"
-require "roast/workflow/base_workflow"
-require "roast/workflow/interpolator"
-
 module Roast
   module Workflow
     # Handles initialization of workflow dependencies: initializers, tools, and API clients
@@ -103,7 +96,7 @@ module Roast
         # Validate the client configuration by making a test API call
         validate_api_client(client) if client
       rescue OpenRouter::ConfigurationError, Faraday::UnauthorizedError => e
-        error = Roast::AuthenticationError.new("API authentication failed: No API token provided or token is invalid")
+        error = Roast::Errors::AuthenticationError.new("API authentication failed: No API token provided or token is invalid")
         error.set_backtrace(e.backtrace)
 
         ActiveSupport::Notifications.instrument("roast.workflow.start.error", {

data/lib/roast/workflow/workflow_runner.rb

@@ -1,12 +1,5 @@
 # frozen_string_literal: true
 
-require "erb"
-require "roast/workflow/replay_handler"
-require "roast/workflow/workflow_executor"
-require "roast/workflow/output_handler"
-require "roast/workflow/base_workflow"
-require "roast/workflow/dot_access_hash"
-
 module Roast
   module Workflow
     # Handles running workflows for files/targets and orchestrating execution
@@ -174,6 +167,12 @@ module Roast
           workflow.verbose = @options[:verbose] if @options[:verbose].present?
           workflow.concise = @options[:concise] if @options[:concise].present?
           workflow.pause_step_name = @options[:pause] if @options[:pause].present?
+          # Set storage type based on CLI option (default is SQLite unless --file-storage is used)
+          workflow.storage_type = @options[:file_storage] ? "file" : nil
+          # Set model from configuration with fallback to default
+          workflow.model = @configuration.model || StepLoader::DEFAULT_MODEL
+          # Set context management configuration
+          workflow.context_management_config = @configuration.context_management
         end
       end
 

data/lib/roast/workflow.rb

@@ -1,20 +1,5 @@
 # frozen_string_literal: true
 
-require "roast/workflow/base_step"
-require "roast/workflow/prompt_step"
-require "roast/workflow/base_iteration_step"
-require "roast/workflow/repeat_step"
-require "roast/workflow/each_step"
-require "roast/workflow/base_workflow"
-require "roast/workflow/configuration"
-require "roast/workflow/workflow_execution_context"
-require "roast/workflow/workflow_executor"
-require "roast/workflow/configuration_parser"
-require "roast/workflow/validator"
-require "roast/workflow/state_repository"
-require "roast/workflow/session_manager"
-require "roast/workflow/file_state_repository"
-
 module Roast
   module Workflow
   end

data/lib/roast/workflow_diagram_generator.rb

@@ -0,0 +1,298 @@
+# frozen_string_literal: true
+
+module Roast
+  class WorkflowDiagramGenerator
+    def initialize(workflow_config, workflow_file_path = nil)
+      @workflow_config = workflow_config
+      @workflow_file_path = workflow_file_path
+      @graph = GraphViz.new(:G, type: :digraph)
+      @node_counter = 0
+      @nodes = {}
+    end
+
+    def generate(custom_output_path = nil)
+      configure_graph
+      build_graph(@workflow_config.steps)
+
+      output_path = custom_output_path || generate_output_filename
+      @graph.output(png: output_path)
+      output_path
+    end
+
+    private
+
+    def configure_graph
+      @graph[:rankdir] = "TB"
+      @graph[:fontname] = "Helvetica"
+      @graph[:fontsize] = "12"
+      @graph[:bgcolor] = "white"
+      @graph[:pad] = "0.5"
+      @graph[:nodesep] = "0.7"
+      @graph[:ranksep] = "0.8"
+      @graph[:splines] = "spline"
+
+      # Default node styling
+      @graph.node[:shape] = "box"
+      @graph.node[:style] = "rounded,filled"
+      @graph.node[:fillcolor] = "#E8F4FD"
+      @graph.node[:color] = "#2563EB"
+      @graph.node[:fontname] = "Helvetica"
+      @graph.node[:fontsize] = "11"
+      @graph.node[:fontcolor] = "#1E293B"
+      @graph.node[:penwidth] = "1.5"
+      @graph.node[:height] = "0.6"
+      @graph.node[:margin] = "0.15"
+
+      # Edge styling
+      @graph.edge[:fontname] = "Helvetica"
+      @graph.edge[:fontsize] = "10"
+      @graph.edge[:color] = "#64748B"
+      @graph.edge[:penwidth] = "1.5"
+      @graph.edge[:arrowsize] = "0.8"
+    end
+
+    def build_graph(steps, parent_node = nil)
+      previous_node = parent_node
+
+      steps.each do |step|
+        current_node = process_step(step)
+
+        if previous_node && current_node
+          @graph.add_edges(previous_node, current_node)
+        end
+
+        previous_node = current_node unless current_node.nil?
+      end
+
+      previous_node
+    end
+
+    def process_step(step)
+      case step
+      when String
+        create_step_node(step)
+      when Hash
+        process_control_flow(step)
+      else
+        ::CLI::Kit.logger.warn("Unexpected step type in workflow diagram: #{step.class} - #{step.inspect}")
+        nil
+      end
+    end
+
+    def create_step_node(step_name)
+      node_id = next_node_id
+      label = step_name
+
+      # Check if it's an inline prompt
+      @nodes[node_id] = if step_name.start_with?("prompt:")
+        @graph.add_nodes(
+          node_id,
+          label: truncate_label(step_name[7..].strip),
+          fillcolor: "#FEF3C7",
+          color: "#F59E0B",
+          shape: "note",
+          fontsize: "10",
+        )
+      else
+        @graph.add_nodes(node_id, label: label)
+      end
+
+      @nodes[node_id]
+    end
+
+    def process_control_flow(control_flow)
+      if control_flow.key?("if") || control_flow.key?("unless")
+        process_conditional(control_flow)
+      elsif control_flow.key?("each") || control_flow.key?("repeat")
+        process_loop(control_flow)
+      elsif control_flow.key?("input")
+        process_input(control_flow)
+      elsif control_flow.key?("proceed?")
+        process_proceed(control_flow)
+      elsif control_flow.key?("case")
+        process_case(control_flow)
+      else
+        ::CLI::Kit.logger.warn("Unexpected control flow structure in workflow diagram: #{control_flow.keys.join(", ")}")
+        nil
+      end
+    end
+
+    def process_conditional(conditional)
+      condition_type = conditional.key?("if") ? "if" : "unless"
+      condition = conditional[condition_type]
+
+      # Create diamond decision node
+      decision_id = next_node_id
+      decision_node = @graph.add_nodes(
+        decision_id,
+        label: "#{condition_type}: #{condition}",
+        shape: "diamond",
+        fillcolor: "#FEE2E2",
+        color: "#DC2626",
+        fontsize: "10",
+        height: "0.8",
+        width: "1.2",
+      )
+
+      # Process then branch
+      if conditional["then"]
+        then_steps = Array(conditional["then"])
+        if then_steps.any?
+          build_graph(then_steps, decision_node)
+        end
+      end
+
+      # Process else branch
+      if conditional["else"]
+        else_steps = Array(conditional["else"])
+        if else_steps.any?
+          build_graph(else_steps, decision_node)
+        end
+      end
+
+      decision_node
+    end
+
+    def process_loop(loop_control)
+      loop_type = loop_control.key?("each") ? "each" : "repeat"
+      loop_value = loop_control[loop_type]
+
+      # Create loop node
+      loop_id = next_node_id
+      loop_label = loop_type == "each" ? "each: #{loop_value}" : "repeat: #{loop_value}"
+      loop_node = @graph.add_nodes(
+        loop_id,
+        label: loop_label,
+        shape: "box3d",
+        fillcolor: "#D1FAE5",
+        color: "#10B981",
+        fontsize: "10",
+        penwidth: "2",
+      )
+
+      # Process loop body
+      if loop_control["do"]
+        loop_steps = Array(loop_control["do"])
+        if loop_steps.any?
+          last_loop_node = build_graph(loop_steps, loop_node)
+          # Add back edge to show loop
+          @graph.add_edges(
+            last_loop_node,
+            loop_node,
+            style: "dashed",
+            label: "loop",
+            color: "#10B981",
+            fontcolor: "#10B981",
+            arrowhead: "empty",
+          )
+        end
+      end
+
+      loop_node
+    end
+
+    def process_input(input_control)
+      input_id = next_node_id
+      label = input_control["input"]
+      input_node = @graph.add_nodes(
+        input_id,
+        label: "input: #{label}",
+        shape: "parallelogram",
+        fillcolor: "#F3F4F6",
+        color: "#6B7280",
+        fontsize: "10",
+      )
+      input_node
+    end
+
+    def process_proceed(proceed_control)
+      proceed_id = next_node_id
+      proceed_node = @graph.add_nodes(
+        proceed_id,
+        label: "proceed?",
+        shape: "diamond",
+        fillcolor: "#FED7AA",
+        color: "#EA580C",
+        fontsize: "10",
+        height: "0.8",
+      )
+
+      # Process do branch if present
+      if proceed_control["do"]
+        proceed_steps = Array(proceed_control["do"])
+        if proceed_steps.any?
+          build_graph(proceed_steps, proceed_node)
+        end
+      end
+
+      proceed_node
+    end
+
+    def process_case(case_control)
+      case_id = next_node_id
+      case_node = @graph.add_nodes(
+        case_id,
+        label: "case: #{case_control["case"]}",
+        shape: "diamond",
+        fillcolor: "#E9D5FF",
+        color: "#9333EA",
+        fontsize: "10",
+        height: "0.8",
+        width: "1.5",
+      )
+
+      # Process when branches
+      case_control["when"].each do |condition, steps|
+        when_steps = Array(steps)
+        next if when_steps.none?
+
+        first_when_node = process_step(when_steps.first)
+        @graph.add_edges(
+          case_node,
+          first_when_node,
+          label: condition.to_s,
+          fontcolor: "#9333EA",
+        )
+
+        if when_steps.length > 1
+          build_graph(when_steps[1..], first_when_node)
+        end
+      end
+
+      case_node
+    end
+
+    def next_node_id
+      @node_counter += 1
+      "node_#{@node_counter}"
+    end
+
+    def truncate_label(text, max_length = 50)
+      return text if text.length <= max_length
+
+      "#{text[0...max_length]}..."
+    end
+
+    def generate_output_filename
+      if @workflow_file_path
+        # Get the directory and base name of the workflow file
+        dir = File.dirname(@workflow_file_path)
+        base = File.basename(@workflow_file_path, ".yml")
+
+        # Create the diagram filename in the same directory
+        File.join(dir, "#{base}.png")
+      else
+        # Fallback to workflow name if no file path provided
+        workflow_name = @workflow_config.name
+        sanitized_name = workflow_name
+          .downcase
+          .gsub(/[^a-z0-9]+/, "_")
+          .gsub(/^_|_$/, "")
+          .gsub(/_+/, "_")
+
+        sanitized_name = "workflow" if sanitized_name.empty?
+        "#{sanitized_name}_diagram.png"
+      end
+    end
+  end
+end