solid_queue_autoscaler 1.0.0

data/lib/solid_queue_autoscaler/metrics.rb

@@ -0,0 +1,172 @@
+# frozen_string_literal: true
+
+module SolidQueueAutoscaler
+  class Metrics
+    Result = Struct.new(
+      :queue_depth,
+      :oldest_job_age_seconds,
+      :jobs_per_minute,
+      :claimed_jobs,
+      :failed_jobs,
+      :blocked_jobs,
+      :active_workers,
+      :queues_breakdown,
+      :collected_at,
+      keyword_init: true
+    ) do
+      def idle?
+        queue_depth.zero? && claimed_jobs.zero?
+      end
+
+      def latency_seconds
+        oldest_job_age_seconds
+      end
+
+      def to_h
+        {
+          queue_depth: queue_depth,
+          oldest_job_age_seconds: oldest_job_age_seconds,
+          jobs_per_minute: jobs_per_minute,
+          claimed_jobs: claimed_jobs,
+          failed_jobs: failed_jobs,
+          blocked_jobs: blocked_jobs,
+          active_workers: active_workers,
+          queues_breakdown: queues_breakdown,
+          collected_at: collected_at
+        }
+      end
+    end
+
+    def initialize(config: nil)
+      @config = config || SolidQueueAutoscaler.config
+    end
+
+    def collect
+      Result.new(
+        queue_depth: queue_depth,
+        oldest_job_age_seconds: oldest_job_age_seconds,
+        jobs_per_minute: jobs_per_minute,
+        claimed_jobs: claimed_jobs_count,
+        failed_jobs: failed_jobs_count,
+        blocked_jobs: blocked_jobs_count,
+        active_workers: active_workers_count,
+        queues_breakdown: queues_breakdown,
+        collected_at: Time.current
+      )
+    end
+
+    def queue_depth
+      sql = <<~SQL
+        SELECT COUNT(*) FROM #{ready_executions_table}
+        WHERE 1=1
+        #{queue_filter_clause}
+      SQL
+      connection.select_value(sql).to_i
+    end
+
+    def oldest_job_age_seconds
+      sql = <<~SQL
+        SELECT EXTRACT(EPOCH FROM (NOW() - MIN(created_at)))
+        FROM #{ready_executions_table}
+        WHERE 1=1
+        #{queue_filter_clause}
+      SQL
+      result = connection.select_value(sql)
+      result.to_f
+    end
+
+    def jobs_per_minute
+      sql = <<~SQL
+        SELECT COUNT(*)
+        FROM #{jobs_table}
+        WHERE finished_at IS NOT NULL
+          AND finished_at > NOW() - INTERVAL '1 minute'
+          #{queue_filter_clause('queue_name')}
+      SQL
+      connection.select_value(sql).to_i
+    end
+
+    def claimed_jobs_count
+      sql = <<~SQL
+        SELECT COUNT(*) FROM #{claimed_executions_table}
+      SQL
+      connection.select_value(sql).to_i
+    end
+
+    def failed_jobs_count
+      sql = <<~SQL
+        SELECT COUNT(*) FROM #{failed_executions_table}
+      SQL
+      connection.select_value(sql).to_i
+    end
+
+    def blocked_jobs_count
+      sql = <<~SQL
+        SELECT COUNT(*) FROM #{blocked_executions_table}
+      SQL
+      connection.select_value(sql).to_i
+    end
+
+    def active_workers_count
+      sql = <<~SQL
+        SELECT COUNT(*)
+        FROM #{processes_table}
+        WHERE kind = 'Worker'
+          AND last_heartbeat_at > NOW() - INTERVAL '5 minutes'
+      SQL
+      connection.select_value(sql).to_i
+    end
+
+    def queues_breakdown
+      sql = <<~SQL
+        SELECT queue_name, COUNT(*) as count
+        FROM #{ready_executions_table}
+        GROUP BY queue_name
+        ORDER BY count DESC
+      SQL
+      connection.select_all(sql).to_a.to_h { |row| [row['queue_name'], row['count'].to_i] }
+    end
+
+    private
+
+    def connection
+      @config.connection
+    end
+
+    def queue_filter_clause(column_name = 'queue_name')
+      return '' unless @config.queues&.any?
+
+      quoted_queues = @config.queues.map { |q| connection.quote(q) }.join(', ')
+      "AND #{column_name} IN (#{quoted_queues})"
+    end
+
+    # Table name helpers using configurable prefix
+    def table_prefix
+      @config.table_prefix
+    end
+
+    def ready_executions_table
+      "#{table_prefix}ready_executions"
+    end
+
+    def jobs_table
+      "#{table_prefix}jobs"
+    end
+
+    def claimed_executions_table
+      "#{table_prefix}claimed_executions"
+    end
+
+    def failed_executions_table
+      "#{table_prefix}failed_executions"
+    end
+
+    def blocked_executions_table
+      "#{table_prefix}blocked_executions"
+    end
+
+    def processes_table
+      "#{table_prefix}processes"
+    end
+  end
+end

data/lib/solid_queue_autoscaler/railtie.rb

@@ -0,0 +1,179 @@
+# frozen_string_literal: true
+
+module SolidQueueAutoscaler
+  class Railtie < Rails::Railtie
+    initializer 'solid_queue_autoscaler.configure' do
+      # Configuration happens via initializer, nothing to do here
+    end
+
+    rake_tasks do
+      namespace :solid_queue_autoscaler do
+        desc 'Run the autoscaler once for a specific worker (default: :default). Use WORKER=name'
+        task scale: :environment do
+          worker_name = (ENV['WORKER'] || 'default').to_sym
+          result = SolidQueueAutoscaler.scale!(worker_name)
+          print_scale_result(result, worker_name)
+        end
+
+        desc 'Run the autoscaler for all configured workers'
+        task scale_all: :environment do
+          results = SolidQueueAutoscaler.scale_all!
+          if results.empty?
+            puts 'No workers configured'
+            exit 1
+          end
+          results.each do |worker_name, result|
+            print_scale_result(result, worker_name)
+          end
+          exit 1 if results.values.any? { |r| !r.success? }
+        end
+
+        desc 'List all configured workers'
+        task workers: :environment do
+          workers = SolidQueueAutoscaler.registered_workers
+          if workers.empty?
+            puts 'No workers configured'
+          else
+            puts "Configured Workers (#{workers.size}):"
+            workers.each do |name|
+              config = SolidQueueAutoscaler.config(name)
+              queues = config.queues&.join(', ') || 'all'
+              puts "  #{name}:"
+              puts "    Process Type: #{config.process_type}"
+              puts "    Queues: #{queues}"
+              puts "    Workers: #{config.min_workers}-#{config.max_workers}"
+            end
+          end
+        end
+
+        desc 'Show current queue metrics for a worker. Use WORKER=name'
+        task metrics: :environment do
+          worker_name = (ENV['WORKER'] || 'default').to_sym
+          metrics = SolidQueueAutoscaler.metrics(worker_name)
+          config = SolidQueueAutoscaler.config(worker_name)
+          puts "Queue Metrics#{" [#{worker_name}]" unless worker_name == :default}:"
+          puts "  Queues Filter: #{config.queues&.join(', ') || 'all'}"
+          puts "  Queue Depth: #{metrics.queue_depth}"
+          puts "  Oldest Job Age: #{metrics.oldest_job_age_seconds.round}s"
+          puts "  Jobs/Minute: #{metrics.jobs_per_minute}"
+          puts "  Claimed Jobs: #{metrics.claimed_jobs}"
+          puts "  Failed Jobs: #{metrics.failed_jobs}"
+          puts "  Blocked Jobs: #{metrics.blocked_jobs}"
+          puts "  Active Workers: #{metrics.active_workers}"
+          puts "  Queues Breakdown: #{metrics.queues_breakdown}"
+        end
+
+        desc 'Show current worker formation. Use WORKER=name'
+        task formation: :environment do
+          worker_name = (ENV['WORKER'] || 'default').to_sym
+          workers = SolidQueueAutoscaler.current_workers(worker_name)
+          config = SolidQueueAutoscaler.config(worker_name)
+          puts "Current Formation#{" [#{worker_name}]" unless worker_name == :default}:"
+          puts "  Process Type: #{config.process_type}"
+          puts "  Workers: #{workers}"
+          puts "  Min: #{config.min_workers}"
+          puts "  Max: #{config.max_workers}"
+          puts "  Queues: #{config.queues&.join(', ') || 'all'}"
+        end
+
+        desc 'Show cooldown state for a worker. Use WORKER=name'
+        task cooldown: :environment do
+          worker_name = (ENV['WORKER'] || 'default').to_sym
+          config = SolidQueueAutoscaler.config(worker_name)
+          tracker = SolidQueueAutoscaler::CooldownTracker.new(config: config, key: worker_name.to_s)
+
+          puts "Cooldown State#{" [#{worker_name}]" unless worker_name == :default}:"
+          puts "  Table Exists: #{tracker.table_exists?}"
+
+          if tracker.table_exists?
+            state = tracker.state
+            puts "  Last Scale Up: #{state[:last_scale_up_at] || 'never'}"
+            puts "  Last Scale Down: #{state[:last_scale_down_at] || 'never'}"
+            puts "  Scale Up Cooldown Active: #{tracker.cooldown_active_for_scale_up?}"
+            puts "  Scale Down Cooldown Active: #{tracker.cooldown_active_for_scale_down?}"
+
+            if tracker.cooldown_active_for_scale_up?
+              puts "  Scale Up Cooldown Remaining: #{tracker.scale_up_cooldown_remaining.round}s"
+            end
+            if tracker.cooldown_active_for_scale_down?
+              puts "  Scale Down Cooldown Remaining: #{tracker.scale_down_cooldown_remaining.round}s"
+            end
+          else
+            puts '  (Using in-memory cooldowns - run migration for persistence)'
+            scale_up = SolidQueueAutoscaler::Scaler.last_scale_up_at(worker_name)
+            scale_down = SolidQueueAutoscaler::Scaler.last_scale_down_at(worker_name)
+            puts "  In-Memory Scale Up: #{scale_up || 'never'}"
+            puts "  In-Memory Scale Down: #{scale_down || 'never'}"
+          end
+        end
+
+        desc 'Reset cooldown state for a worker (or all if WORKER=all). Use WORKER=name'
+        task reset_cooldown: :environment do
+          worker_name = ENV.fetch('WORKER', nil)&.to_sym
+
+          if worker_name == :all || worker_name.nil?
+            # Reset all workers
+            SolidQueueAutoscaler.registered_workers.each do |name|
+              config = SolidQueueAutoscaler.config(name)
+              tracker = SolidQueueAutoscaler::CooldownTracker.new(config: config, key: name.to_s)
+              tracker.reset! if tracker.table_exists?
+            end
+            SolidQueueAutoscaler::Scaler.reset_cooldowns!
+            puts 'All cooldown states reset'
+          else
+            config = SolidQueueAutoscaler.config(worker_name)
+            tracker = SolidQueueAutoscaler::CooldownTracker.new(config: config, key: worker_name.to_s)
+            tracker.reset! if tracker.table_exists?
+            SolidQueueAutoscaler::Scaler.reset_cooldowns!(worker_name)
+            puts "Cooldown state reset for #{worker_name}"
+          end
+        end
+
+        desc 'Show recent scale events. Use LIMIT=n and WORKER=name'
+        task events: :environment do
+          worker_name = ENV.fetch('WORKER', nil)
+          limit = (ENV['LIMIT'] || 20).to_i
+
+          events = SolidQueueAutoscaler::ScaleEvent.recent(limit: limit, worker_name: worker_name)
+
+          if events.empty?
+            puts 'No events found'
+            puts '(Make sure to run: rails generate solid_queue_autoscaler:dashboard)'
+          else
+            puts "Recent Scale Events#{" for #{worker_name}" if worker_name} (#{events.size}):"
+            puts '-' * 100
+            events.each do |event|
+              action = event.action.ljust(10)
+              workers = "#{event.from_workers}->#{event.to_workers}".ljust(8)
+              dry_run = event.dry_run ? ' [DRY RUN]' : ''
+              time = event.created_at.strftime('%Y-%m-%d %H:%M:%S')
+              puts "#{time} | #{event.worker_name.ljust(15)} | #{action} | #{workers} | #{event.reason}#{dry_run}"
+            end
+          end
+        end
+
+        desc 'Cleanup old scale events. Use KEEP_DAYS=n (default: 30)'
+        task cleanup_events: :environment do
+          keep_days = (ENV['KEEP_DAYS'] || 30).to_i
+          SolidQueueAutoscaler::ScaleEvent.cleanup!(keep_days: keep_days)
+          puts "Cleaned up events older than #{keep_days} days"
+        end
+
+        def print_scale_result(result, worker_name)
+          prefix = worker_name == :default ? '' : "[#{worker_name}] "
+          if result.success?
+            if result.scaled?
+              puts "#{prefix}Scaled #{result.decision.from} -> #{result.decision.to} workers"
+            elsif result.skipped?
+              puts "#{prefix}Skipped: #{result.skipped_reason}"
+            else
+              puts "#{prefix}No change needed: #{result.decision&.reason}"
+            end
+          else
+            puts "#{prefix}Error: #{result.error}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/solid_queue_autoscaler/scale_event.rb

@@ -0,0 +1,292 @@
+# frozen_string_literal: true
+
+module SolidQueueAutoscaler
+  # Lightweight model for recording autoscaler events.
+  # Does not inherit from ActiveRecord to avoid requiring it as a dependency.
+  # Uses raw SQL for compatibility with any database connection.
+  class ScaleEvent
+    TABLE_NAME = 'solid_queue_autoscaler_events'
+
+    ACTIONS = %w[scale_up scale_down no_change skipped error].freeze
+
+    attr_reader :id, :worker_name, :action, :from_workers, :to_workers,
+                :reason, :queue_depth, :latency_seconds, :metrics_json,
+                :dry_run, :created_at
+
+    def initialize(attrs = {})
+      @id = attrs[:id]
+      @worker_name = attrs[:worker_name]
+      @action = attrs[:action]
+      @from_workers = attrs[:from_workers]
+      @to_workers = attrs[:to_workers]
+      @reason = attrs[:reason]
+      @queue_depth = attrs[:queue_depth]
+      @latency_seconds = attrs[:latency_seconds]
+      @metrics_json = attrs[:metrics_json]
+      @dry_run = attrs[:dry_run]
+      @created_at = attrs[:created_at]
+    end
+
+    def scaled?
+      %w[scale_up scale_down].include?(action)
+    end
+
+    def scale_up?
+      action == 'scale_up'
+    end
+
+    def scale_down?
+      action == 'scale_down'
+    end
+
+    def metrics
+      return nil unless metrics_json
+
+      JSON.parse(metrics_json, symbolize_names: true)
+    rescue JSON::ParserError
+      nil
+    end
+
+    class << self
+      # Creates a new scale event record.
+      # @param attrs [Hash] Event attributes
+      # @param connection [ActiveRecord::ConnectionAdapters::AbstractAdapter] Database connection
+      # @return [ScaleEvent] The created event
+      def create!(attrs, connection: nil)
+        conn = connection || default_connection
+        return nil unless table_exists?(conn)
+
+        now = Time.current
+        sql = <<~SQL
+          INSERT INTO #{TABLE_NAME}
+            (worker_name, action, from_workers, to_workers, reason,
+             queue_depth, latency_seconds, metrics_json, dry_run, created_at)
+          VALUES
+            (#{conn.quote(attrs[:worker_name])},
+             #{conn.quote(attrs[:action])},
+             #{conn.quote(attrs[:from_workers])},
+             #{conn.quote(attrs[:to_workers])},
+             #{conn.quote(attrs[:reason])},
+             #{conn.quote(attrs[:queue_depth])},
+             #{conn.quote(attrs[:latency_seconds])},
+             #{conn.quote(attrs[:metrics_json])},
+             #{conn.quote(attrs[:dry_run])},
+             #{conn.quote(now)})
+          RETURNING id
+        SQL
+
+        result = conn.execute(sql)
+        id = result.first&.fetch('id', nil)
+
+        new(attrs.merge(id: id, created_at: now))
+      rescue StandardError => e
+        # Log but don't fail if event recording fails
+        Rails.logger.warn("[Autoscaler] Failed to record event: #{e.message}") if defined?(Rails)
+        nil
+      end
+
+      # Finds recent events.
+      # @param limit [Integer] Maximum number of events to return
+      # @param worker_name [String, nil] Filter by worker name
+      # @param connection [ActiveRecord::ConnectionAdapters::AbstractAdapter] Database connection
+      # @return [Array<ScaleEvent>] Array of events
+      def recent(limit: 50, worker_name: nil, connection: nil)
+        conn = connection || default_connection
+        return [] unless table_exists?(conn)
+
+        filter = worker_name ? "WHERE worker_name = #{conn.quote(worker_name)}" : ''
+
+        sql = <<~SQL
+          SELECT id, worker_name, action, from_workers, to_workers, reason,
+                 queue_depth, latency_seconds, metrics_json, dry_run, created_at
+          FROM #{TABLE_NAME}
+          #{filter}
+          ORDER BY created_at DESC
+          LIMIT #{limit.to_i}
+        SQL
+
+        conn.select_all(sql).map { |row| from_row(row) }
+      rescue StandardError
+        []
+      end
+
+      # Finds events by action type.
+      # @param action [String] Action type (scale_up, scale_down, etc.)
+      # @param limit [Integer] Maximum number of events
+      # @param connection [ActiveRecord::ConnectionAdapters::AbstractAdapter] Database connection
+      # @return [Array<ScaleEvent>] Array of events
+      def by_action(action, limit: 50, connection: nil)
+        conn = connection || default_connection
+        return [] unless table_exists?(conn)
+
+        sql = <<~SQL
+          SELECT id, worker_name, action, from_workers, to_workers, reason,
+                 queue_depth, latency_seconds, metrics_json, dry_run, created_at
+          FROM #{TABLE_NAME}
+          WHERE action = #{conn.quote(action)}
+          ORDER BY created_at DESC
+          LIMIT #{limit.to_i}
+        SQL
+
+        conn.select_all(sql).map { |row| from_row(row) }
+      rescue StandardError
+        []
+      end
+
+      # Gets event statistics for a time period.
+      # @param since [Time] Start time for statistics
+      # @param worker_name [String, nil] Filter by worker name
+      # @param connection [ActiveRecord::ConnectionAdapters::AbstractAdapter] Database connection
+      # @return [Hash] Statistics hash
+      def stats(since: 24.hours.ago, worker_name: nil, connection: nil)
+        conn = connection || default_connection
+        return default_stats unless table_exists?(conn)
+
+        worker_filter = worker_name ? "AND worker_name = #{conn.quote(worker_name)}" : ''
+
+        sql = <<~SQL
+          SELECT
+            action,
+            COUNT(*) as count,
+            AVG(queue_depth) as avg_queue_depth,
+            AVG(latency_seconds) as avg_latency
+          FROM #{TABLE_NAME}
+          WHERE created_at >= #{conn.quote(since)}
+          #{worker_filter}
+          GROUP BY action
+        SQL
+
+        results = conn.select_all(sql).to_a
+        build_stats(results)
+      rescue StandardError
+        default_stats
+      end
+
+      # Cleans up old events.
+      # @param keep_days [Integer] Number of days to keep
+      # @param connection [ActiveRecord::ConnectionAdapters::AbstractAdapter] Database connection
+      # @return [Integer] Number of deleted records
+      def cleanup!(keep_days: 30, connection: nil)
+        conn = connection || default_connection
+        return 0 unless table_exists?(conn)
+
+        cutoff = Time.current - keep_days.days
+
+        sql = <<~SQL
+          DELETE FROM #{TABLE_NAME}
+          WHERE created_at < #{conn.quote(cutoff)}
+        SQL
+
+        result = conn.execute(sql)
+        result.cmd_tuples
+      rescue StandardError
+        0
+      end
+
+      # Checks if the events table exists.
+      # @param connection [ActiveRecord::ConnectionAdapters::AbstractAdapter] Database connection
+      # @return [Boolean] True if table exists
+      def table_exists?(connection = nil)
+        conn = connection || default_connection
+        conn.table_exists?(TABLE_NAME)
+      rescue StandardError
+        false
+      end
+
+      # Counts events in a time period.
+      # @param since [Time] Start time
+      # @param connection [ActiveRecord::ConnectionAdapters::AbstractAdapter] Database connection
+      # @return [Integer] Event count
+      def count(since: nil, connection: nil)
+        conn = connection || default_connection
+        return 0 unless table_exists?(conn)
+
+        time_filter = since ? "WHERE created_at >= #{conn.quote(since)}" : ''
+
+        sql = "SELECT COUNT(*) FROM #{TABLE_NAME} #{time_filter}"
+        conn.select_value(sql).to_i
+      rescue StandardError
+        0
+      end
+
+      private
+
+      def default_connection
+        ActiveRecord::Base.connection
+      end
+
+      def from_row(row)
+        new(
+          id: row['id'],
+          worker_name: row['worker_name'],
+          action: row['action'],
+          from_workers: row['from_workers'].to_i,
+          to_workers: row['to_workers'].to_i,
+          reason: row['reason'],
+          queue_depth: row['queue_depth'].to_i,
+          latency_seconds: row['latency_seconds'].to_f,
+          metrics_json: row['metrics_json'],
+          dry_run: parse_boolean(row['dry_run']),
+          created_at: parse_time(row['created_at'])
+        )
+      end
+
+      def parse_boolean(value)
+        case value
+        when true, 't', 'true', '1', 1
+          true
+        else
+          false
+        end
+      end
+
+      def parse_time(value)
+        case value
+        when Time, DateTime
+          value.to_time
+        when String
+          Time.parse(value)
+        else
+          value
+        end
+      end
+
+      def default_stats
+        {
+          total: 0,
+          scale_up_count: 0,
+          scale_down_count: 0,
+          no_change_count: 0,
+          skipped_count: 0,
+          error_count: 0,
+          avg_queue_depth: 0,
+          avg_latency: 0
+        }
+      end
+
+      def build_stats(results)
+        stats = default_stats
+
+        results.each do |row|
+          action = row['action']
+          count = row['count'].to_i
+
+          stats[:total] += count
+          stats[:"#{action}_count"] = count
+
+          # Use weighted average for overall metrics
+          stats[:avg_queue_depth] += row['avg_queue_depth'].to_f * count if row['avg_queue_depth']
+          stats[:avg_latency] += row['avg_latency'].to_f * count if row['avg_latency']
+        end
+
+        # Calculate averages
+        if stats[:total].positive?
+          stats[:avg_queue_depth] /= stats[:total]
+          stats[:avg_latency] /= stats[:total]
+        end
+
+        stats
+      end
+    end
+  end
+end