roast-ai 0.4.0 → 0.4.1

data/lib/roast/workflow/sqlite_state_repository.rb

@@ -0,0 +1,342 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Roast
+  module Workflow
+    # SQLite-based implementation of StateRepository
+    # Provides structured, queryable session storage with better performance
+    class SqliteStateRepository < StateRepository
+      DEFAULT_DB_PATH = File.expand_path("~/.roast/sessions.db")
+
+      def initialize(db_path: nil, session_manager: SessionManager.new)
+        super()
+
+        # Lazy load sqlite3 only when actually using SQLite storage
+        begin
+          require "sqlite3"
+        rescue LoadError
+          raise LoadError, "SQLite storage requires the 'sqlite3' gem. Please add it to your Gemfile or install it: gem install sqlite3"
+        end
+
+        @db_path = db_path || ENV["ROAST_SESSIONS_DB"] || DEFAULT_DB_PATH
+        @session_manager = session_manager
+        ensure_database
+      end
+
+      def save_state(workflow, step_name, state_data)
+        workflow.session_timestamp ||= @session_manager.create_new_session(workflow.object_id)
+
+        session_id = ensure_session(workflow)
+
+        @db.execute(<<~SQL, session_id, state_data[:order], step_name, state_data.to_json)
+          INSERT INTO session_states (session_id, step_index, step_name, state_data)
+          VALUES (?, ?, ?, ?)
+        SQL
+
+        # Update session's current step
+        @db.execute(<<~SQL, state_data[:order], session_id)
+          UPDATE sessions#{" "}
+          SET current_step_index = ?, updated_at = CURRENT_TIMESTAMP
+          WHERE id = ?
+        SQL
+      rescue => e
+        $stderr.puts "Failed to save state for step #{step_name}: #{e.message}"
+      end
+
+      def load_state_before_step(workflow, step_name, timestamp: nil)
+        session_id = find_session_id(workflow, timestamp)
+        return false unless session_id
+
+        # Find the state before the target step
+        result = @db.execute(<<~SQL, session_id, step_name)
+          SELECT state_data, step_name
+          FROM session_states
+          WHERE session_id = ?
+            AND step_index < (
+              SELECT MIN(step_index)#{" "}
+              FROM session_states#{" "}
+              WHERE session_id = ? AND step_name = ?
+            )
+          ORDER BY step_index DESC
+          LIMIT 1
+        SQL
+
+        if result.empty?
+          # Try to find the latest state if target step doesn't exist
+          result = @db.execute(<<~SQL, session_id)
+            SELECT state_data, step_name
+            FROM session_states
+            WHERE session_id = ?
+            ORDER BY step_index DESC
+            LIMIT 1
+          SQL
+
+          if result.empty?
+            $stderr.puts "No state found for session"
+            return false
+          end
+        end
+
+        state_data = JSON.parse(result[0][0], symbolize_names: true)
+        loaded_step = result[0][1]
+        $stderr.puts "Found state from step: #{loaded_step} (will replay from here to #{step_name})"
+
+        # If no timestamp provided and workflow has no session, create new session and copy states
+        if !timestamp && workflow.session_timestamp.nil?
+          copy_states_to_new_session(workflow, session_id, step_name)
+        end
+
+        state_data
+      end
+
+      def save_final_output(workflow, output_content)
+        return if output_content.empty?
+
+        session_id = ensure_session(workflow)
+
+        @db.execute(<<~SQL, output_content, session_id)
+          UPDATE sessions#{" "}
+          SET final_output = ?, status = 'completed', updated_at = CURRENT_TIMESTAMP
+          WHERE id = ?
+        SQL
+
+        session_id
+      rescue => e
+        $stderr.puts "Failed to save final output: #{e.message}"
+        nil
+      end
+
+      # Additional query methods for the new capabilities
+
+      def list_sessions(status: nil, workflow_name: nil, older_than: nil, limit: 100)
+        conditions = []
+        params = []
+
+        if status
+          conditions << "status = ?"
+          params << status
+        end
+
+        if workflow_name
+          conditions << "workflow_name = ?"
+          params << workflow_name
+        end
+
+        if older_than
+          conditions << "created_at < datetime('now', ?)"
+          params << "-#{older_than}"
+        end
+
+        where_clause = conditions.empty? ? "" : "WHERE #{conditions.join(" AND ")}"
+
+        @db.execute(<<~SQL, *params)
+          SELECT id, workflow_name, workflow_path, status, current_step_index,#{" "}
+                 created_at, updated_at
+          FROM sessions
+          #{where_clause}
+          ORDER BY created_at DESC
+          LIMIT #{limit}
+        SQL
+      end
+
+      def get_session_details(session_id)
+        session = @db.execute(<<~SQL, session_id).first
+          SELECT * FROM sessions WHERE id = ?
+        SQL
+
+        return unless session
+
+        states = @db.execute(<<~SQL, session_id)
+          SELECT step_index, step_name, created_at
+          FROM session_states
+          WHERE session_id = ?
+          ORDER BY step_index
+        SQL
+
+        events = @db.execute(<<~SQL, session_id)
+          SELECT event_name, event_data, received_at
+          FROM session_events
+          WHERE session_id = ?
+          ORDER BY received_at
+        SQL
+
+        {
+          session: session,
+          states: states,
+          events: events,
+        }
+      end
+
+      def cleanup_old_sessions(older_than)
+        count = @db.changes
+        @db.execute(<<~SQL, "-#{older_than}")
+          DELETE FROM sessions
+          WHERE created_at < datetime('now', ?)
+        SQL
+        @db.changes - count
+      end
+
+      def add_event(workflow_path, session_id, event_name, event_data = nil)
+        # Find the session if session_id not provided
+        unless session_id
+          workflow_name = File.basename(File.dirname(workflow_path))
+          result = @db.execute(<<~SQL, workflow_name, "waiting")
+            SELECT id FROM sessions
+            WHERE workflow_name = ? AND status = ?
+            ORDER BY created_at DESC
+            LIMIT 1
+          SQL
+
+          raise "No waiting session found for workflow: #{workflow_name}" if result.empty?
+
+          session_id = result[0][0]
+        end
+
+        # Add the event
+        @db.execute(<<~SQL, session_id, event_name, event_data&.to_json)
+          INSERT INTO session_events (session_id, event_name, event_data)
+          VALUES (?, ?, ?)
+        SQL
+
+        # Update session status
+        @db.execute(<<~SQL, session_id)
+          UPDATE sessions#{" "}
+          SET status = 'running', updated_at = CURRENT_TIMESTAMP
+          WHERE id = ?
+        SQL
+
+        session_id
+      end
+
+      private
+
+      def ensure_database
+        FileUtils.mkdir_p(File.dirname(@db_path))
+        @db = SQLite3::Database.new(@db_path)
+        @db.execute("PRAGMA foreign_keys = ON")
+        create_schema
+      end
+
+      def create_schema
+        @db.execute_batch(<<~SQL)
+          CREATE TABLE IF NOT EXISTS sessions (
+            id TEXT PRIMARY KEY,
+            workflow_name TEXT NOT NULL,
+            workflow_path TEXT NOT NULL,
+            status TEXT NOT NULL DEFAULT 'running',
+            current_step_index INTEGER,
+            final_output TEXT,
+            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+            updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+          );
+
+          CREATE TABLE IF NOT EXISTS session_states (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            session_id TEXT NOT NULL,
+            step_index INTEGER NOT NULL,
+            step_name TEXT NOT NULL,
+            state_data TEXT NOT NULL,
+            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+            FOREIGN KEY (session_id) REFERENCES sessions(id) ON DELETE CASCADE
+          );
+
+          CREATE TABLE IF NOT EXISTS session_events (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            session_id TEXT NOT NULL,
+            event_name TEXT NOT NULL,
+            event_data TEXT,
+            received_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+            FOREIGN KEY (session_id) REFERENCES sessions(id) ON DELETE CASCADE
+          );
+
+          CREATE TABLE IF NOT EXISTS session_variables (
+            session_id TEXT NOT NULL,
+            key TEXT NOT NULL,
+            value TEXT NOT NULL,
+            updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+            PRIMARY KEY (session_id, key),
+            FOREIGN KEY (session_id) REFERENCES sessions(id) ON DELETE CASCADE
+          );
+
+          -- Indexes for common queries
+          CREATE INDEX IF NOT EXISTS idx_sessions_status ON sessions(status);
+          CREATE INDEX IF NOT EXISTS idx_sessions_workflow_name ON sessions(workflow_name);
+          CREATE INDEX IF NOT EXISTS idx_sessions_created_at ON sessions(created_at);
+          CREATE INDEX IF NOT EXISTS idx_session_states_session_id ON session_states(session_id);
+          CREATE INDEX IF NOT EXISTS idx_session_events_session_id ON session_events(session_id);
+        SQL
+      end
+
+      def ensure_session(workflow)
+        session_id = generate_session_id(workflow)
+
+        # Check if session exists
+        existing = @db.execute("SELECT id FROM sessions WHERE id = ?", session_id).first
+        return session_id if existing
+
+        # Create new session
+        workflow_name = workflow.session_name || "unnamed"
+        workflow_path = workflow.file || "notarget"
+
+        @db.execute(<<~SQL, session_id, workflow_name, workflow_path)
+          INSERT INTO sessions (id, workflow_name, workflow_path)
+          VALUES (?, ?, ?)
+        SQL
+
+        session_id
+      end
+
+      def find_session_id(workflow, timestamp)
+        if timestamp
+          # Find by exact timestamp
+          generate_session_id(workflow, timestamp)
+        else
+          # Find latest session for this workflow
+          workflow_name = workflow.session_name || "unnamed"
+          workflow_path = workflow.file || "notarget"
+
+          result = @db.execute(<<~SQL, workflow_name, workflow_path)
+            SELECT id FROM sessions
+            WHERE workflow_name = ? AND workflow_path = ?
+            ORDER BY created_at DESC
+            LIMIT 1
+          SQL
+
+          result.empty? ? nil : result[0][0]
+        end
+      end
+
+      def generate_session_id(workflow, timestamp = nil)
+        timestamp ||= workflow.session_timestamp || @session_manager.create_new_session(workflow.object_id)
+        workflow_name = workflow.session_name || "unnamed"
+        workflow_path = workflow.file || "notarget"
+
+        # Generate a unique session ID based on workflow info and timestamp
+        file_hash = Digest::MD5.hexdigest(workflow_path)[0..7]
+        "#{workflow_name.parameterize.underscore}_#{file_hash}_#{timestamp}"
+      end
+
+      def copy_states_to_new_session(workflow, source_session_id, target_step_name)
+        # Create new session
+        new_timestamp = @session_manager.create_new_session(workflow.object_id)
+        workflow.session_timestamp = new_timestamp
+        new_session_id = ensure_session(workflow)
+
+        # Copy states up to the target step
+        @db.execute(<<~SQL, new_session_id, source_session_id, target_step_name, source_session_id)
+          INSERT INTO session_states (session_id, step_index, step_name, state_data)
+          SELECT ?, step_index, step_name, state_data
+          FROM session_states
+          WHERE session_id = ?
+            AND step_index < COALESCE(
+              (SELECT MIN(step_index) FROM session_states WHERE session_id = ? AND step_name = ?),
+              999999
+            )
+        SQL
+
+        true
+      end
+    end
+  end
+end

data/lib/roast/workflow/state_manager.rb

@@ -6,10 +6,10 @@ module Roast
     class StateManager
       attr_reader :workflow, :logger
 
-      def initialize(workflow, logger: nil)
+      def initialize(workflow, logger: nil, state_repository: nil, storage_type: nil)
         @workflow = workflow
         @logger = logger
-        @state_repository = FileStateRepository.new
+        @state_repository = state_repository || StateRepositoryFactory.create(storage_type)
       end
 
       # Save the current state after a step execution

data/lib/roast/workflow/state_repository_factory.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Roast
+  module Workflow
+    # Factory for creating the appropriate StateRepository implementation
+    class StateRepositoryFactory
+      class << self
+        def create(type = nil)
+          type ||= default_type
+
+          case type.to_s
+          when "sqlite"
+            # Lazy load the SQLite repository only when needed
+            Roast::Workflow::SqliteStateRepository.new
+          when "file", "filesystem"
+            Roast::Workflow::FileStateRepository.new
+          else
+            raise ArgumentError, "Unknown state repository type: #{type}. Valid types are: sqlite, file"
+          end
+        end
+
+        private
+
+        def default_type
+          # Check environment variable first (for backwards compatibility)
+          if ENV["ROAST_STATE_STORAGE"]
+            ENV["ROAST_STATE_STORAGE"].downcase
+          else
+            # Default to SQLite for better functionality
+            "sqlite"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/roast/workflow/step_completion_reporter.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Roast
+  module Workflow
+    # Reports step completion with token consumption information
+    class StepCompletionReporter
+      def initialize(output: $stderr)
+        @output = output
+      end
+
+      def report(step_name, tokens_consumed, total_tokens)
+        formatted_consumed = number_with_delimiter(tokens_consumed)
+        formatted_total = number_with_delimiter(total_tokens)
+
+        @output.puts "✓ Complete: #{step_name} (consumed #{formatted_consumed} tokens, total #{formatted_total})"
+        @output.puts
+        @output.puts
+      end
+
+      private
+
+      def number_with_delimiter(number)
+        number.to_s.gsub(/(\d)(?=(\d{3})+(?!\d))/, '\\1,')
+      end
+    end
+  end
+end

data/lib/roast/workflow/step_executor_coordinator.rb

@@ -28,11 +28,11 @@ module Roast
           is_last_step = (index == workflow_steps.length - 1)
           case step
           when Hash
-            execute(step, is_last_step: is_last_step)
+            execute(step, is_last_step:)
           when Array
-            execute(step, is_last_step: is_last_step)
+            execute(step, is_last_step:)
           when String
-            execute(step, is_last_step: is_last_step)
+            execute(step, is_last_step:)
             # Handle pause after string steps
             if @context.workflow.pause_step_name == step
               Kernel.binding.irb # rubocop:disable Lint/Debugger
@@ -62,17 +62,17 @@ module Roast
         when StepTypeResolver::AGENT_STEP
           execute_agent_step(step, options)
         when StepTypeResolver::GLOB_STEP
-          execute_glob_step(step)
+          execute_glob_step(step, options)
         when StepTypeResolver::ITERATION_STEP
-          execute_iteration_step(step)
+          execute_iteration_step(step, options)
         when StepTypeResolver::CONDITIONAL_STEP
-          execute_conditional_step(step)
+          execute_conditional_step(step, options)
         when StepTypeResolver::CASE_STEP
-          execute_case_step(step)
+          execute_case_step(step, options)
         when StepTypeResolver::INPUT_STEP
-          execute_input_step(step)
+          execute_input_step(step, options)
         when StepTypeResolver::HASH_STEP
-          execute_hash_step(step)
+          execute_hash_step(step, options)
         when StepTypeResolver::PARALLEL_STEP
           # Use factory for parallel steps
           executor = StepExecutorFactory.for(step, workflow_executor)
@@ -189,11 +189,11 @@ module Roast
         step_orchestrator.execute_step(step_name, exit_on_error:, step_key: options[:step_key], agent_type: :coding_agent)
       end
 
-      def execute_glob_step(step)
+      def execute_glob_step(step, options = {})
         Dir.glob(step).join("\n")
       end
 
-      def execute_iteration_step(step)
+      def execute_iteration_step(step, options = {})
         name = step.keys.first
         command = step[name]
 
@@ -206,19 +206,19 @@ module Roast
         end
       end
 
-      def execute_conditional_step(step)
+      def execute_conditional_step(step, options = {})
         conditional_executor.execute_conditional(step)
       end
 
-      def execute_case_step(step)
+      def execute_case_step(step, options = {})
         case_executor.execute_case(step)
       end
 
-      def execute_input_step(step)
+      def execute_input_step(step, options = {})
         input_executor.execute_input(step["input"])
       end
 
-      def execute_hash_step(step)
+      def execute_hash_step(step, options = {})
         name, command = step.to_a.flatten
         interpolated_name = interpolator.interpolate(name)
 
@@ -230,7 +230,8 @@ module Roast
 
           # Execute the command directly using the appropriate executor
           # Pass the original key name for configuration lookup
-          result = execute(interpolated_command, { exit_on_error: exit_on_error, step_key: interpolated_name })
+          # Merge options to preserve is_last_step
+          result = execute(interpolated_command, { exit_on_error:, step_key: interpolated_name }.merge(options))
           context.workflow.output[interpolated_name] = result
           result
         end
@@ -247,10 +248,10 @@ module Roast
         if StepTypeResolver.command_step?(interpolated_step)
           # Command step - execute directly, preserving any passed options
           exit_on_error = options.fetch(:exit_on_error, true)
-          execute_command_step(interpolated_step, { exit_on_error: exit_on_error })
+          execute_command_step(interpolated_step, { exit_on_error: })
         else
           exit_on_error = options.fetch(:exit_on_error, context.exit_on_error?(step))
-          execute_standard_step(interpolated_step, options.merge(exit_on_error: exit_on_error))
+          execute_standard_step(interpolated_step, options.merge(exit_on_error:))
         end
       end
 

data/lib/roast/workflow/step_executor_with_reporting.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Roast
+  module Workflow
+    # Decorator that adds token consumption reporting to step execution
+    class StepExecutorWithReporting
+      def initialize(base_executor, context, output: $stderr)
+        @base_executor = base_executor
+        @context = context
+        @reporter = StepCompletionReporter.new(output: output)
+        @name_extractor = StepNameExtractor.new
+      end
+
+      def execute(step, options = {})
+        # Track tokens before execution
+        tokens_before = @context.workflow.context_manager&.total_tokens || 0
+
+        # Execute the step
+        result = @base_executor.execute(step, options)
+
+        # Report token consumption after successful execution
+        tokens_after = @context.workflow.context_manager&.total_tokens || 0
+        tokens_consumed = tokens_after - tokens_before
+
+        step_type = StepTypeResolver.resolve(step, @context)
+        step_name = @name_extractor.extract(step, step_type)
+        @reporter.report(step_name, tokens_consumed, tokens_after)
+
+        result
+      end
+
+      # Override execute_steps to ensure reporting happens for each step
+      def execute_steps(workflow_steps)
+        workflow_steps.each_with_index do |step, index|
+          is_last_step = (index == workflow_steps.length - 1)
+          case step
+          when Hash
+            execute(step, is_last_step:)
+          when Array
+            execute(step, is_last_step:)
+          when String
+            execute(step, is_last_step:)
+            # Handle pause after string steps
+            if @context.workflow.pause_step_name == step
+              Kernel.binding.irb # rubocop:disable Lint/Debugger
+            end
+          else
+            # For other types, delegate to base executor
+            execute(step, is_last_step:)
+          end
+        end
+      end
+
+      # Delegate all other methods to the base executor
+      def method_missing(method, *args, **kwargs, &block)
+        if @base_executor.respond_to?(method)
+          @base_executor.send(method, *args, **kwargs, &block)
+        else
+          super
+        end
+      end
+
+      def respond_to_missing?(method, include_private = false)
+        @base_executor.respond_to?(method, include_private) || super
+      end
+    end
+  end
+end

data/lib/roast/workflow/step_loader.rb

@@ -4,7 +4,7 @@ module Roast
   module Workflow
     # Handles loading and instantiation of workflow steps
     class StepLoader
-      DEFAULT_MODEL = "openai/gpt-4o-mini"
+      DEFAULT_MODEL = "gpt-4o-mini"
 
       # Custom exception classes
       class StepLoaderError < StandardError

data/lib/roast/workflow/step_name_extractor.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Roast
+  module Workflow
+    # Extracts human-readable names from various step types
+    class StepNameExtractor
+      def extract(step, step_type)
+        case step_type
+        when StepTypeResolver::COMMAND_STEP
+          extract_command_name(step)
+        when StepTypeResolver::HASH_STEP
+          extract_hash_step_name(step)
+        when StepTypeResolver::ITERATION_STEP
+          extract_iteration_step_name(step)
+        when StepTypeResolver::CONDITIONAL_STEP
+          extract_conditional_step_name(step)
+        when StepTypeResolver::CASE_STEP
+          "case"
+        when StepTypeResolver::INPUT_STEP
+          "input"
+        when StepTypeResolver::AGENT_STEP
+          StepTypeResolver.extract_name(step)
+        when StepTypeResolver::STRING_STEP
+          step.to_s
+        else
+          step.to_s
+        end
+      end
+
+      private
+
+      def extract_command_name(step)
+        cmd = step.to_s.strip
+        cmd.length > 20 ? "#{cmd[0..19]}..." : cmd
+      end
+
+      def extract_hash_step_name(step)
+        key, value = step.to_a.first
+
+        # Check if this looks like an inline prompt (key is similar to sanitized value)
+        if value.is_a?(String)
+          # Get first non-empty line
+          first_line = value.lines.map(&:strip).find { |line| !line.empty? } || ""
+
+          # If key looks like it was auto-generated from the content, use truncated content
+          sanitized = first_line.downcase.gsub(/[^a-z0-9_]/, "_").squeeze("_").gsub(/^_|_$/, "")
+          if key.to_s == sanitized || key.to_s.start_with?(sanitized[0..15])
+            # This is likely an inline prompt
+            first_line.length > 20 ? "#{first_line[0..19]}..." : first_line
+          else
+            # This is a labeled step
+            key.to_s
+          end
+        else
+          key.to_s
+        end
+      end
+
+      def extract_iteration_step_name(step)
+        if step.key?("each")
+          items = step["each"]
+          count = items.respond_to?(:size) ? items.size : "?"
+          "each (#{count} items)"
+        elsif step.key?("repeat")
+          config = step["repeat"]
+          times = config.is_a?(Hash) ? config["times"] || "?" : config
+          "repeat (#{times} times)"
+        else
+          "iteration"
+        end
+      end
+
+      def extract_conditional_step_name(step)
+        if step.key?("if")
+          "if"
+        elsif step.key?("unless")
+          "unless"
+        else
+          "conditional"
+        end
+      end
+    end
+  end
+end