solid_queue_heroku_autoscaler 0.1.0

data/lib/solid_queue_heroku_autoscaler/metrics.rb

@@ -0,0 +1,172 @@
+# frozen_string_literal: true
+
+module SolidQueueHerokuAutoscaler
+  class Metrics
+    Result = Struct.new(
+      :queue_depth,
+      :oldest_job_age_seconds,
+      :jobs_per_minute,
+      :claimed_jobs,
+      :failed_jobs,
+      :blocked_jobs,
+      :active_workers,
+      :queues_breakdown,
+      :collected_at,
+      keyword_init: true
+    ) do
+      def idle?
+        queue_depth.zero? && claimed_jobs.zero?
+      end
+
+      def latency_seconds
+        oldest_job_age_seconds
+      end
+
+      def to_h
+        {
+          queue_depth: queue_depth,
+          oldest_job_age_seconds: oldest_job_age_seconds,
+          jobs_per_minute: jobs_per_minute,
+          claimed_jobs: claimed_jobs,
+          failed_jobs: failed_jobs,
+          blocked_jobs: blocked_jobs,
+          active_workers: active_workers,
+          queues_breakdown: queues_breakdown,
+          collected_at: collected_at
+        }
+      end
+    end
+
+    def initialize(config: nil)
+      @config = config || SolidQueueHerokuAutoscaler.config
+    end
+
+    def collect
+      Result.new(
+        queue_depth: queue_depth,
+        oldest_job_age_seconds: oldest_job_age_seconds,
+        jobs_per_minute: jobs_per_minute,
+        claimed_jobs: claimed_jobs_count,
+        failed_jobs: failed_jobs_count,
+        blocked_jobs: blocked_jobs_count,
+        active_workers: active_workers_count,
+        queues_breakdown: queues_breakdown,
+        collected_at: Time.current
+      )
+    end
+
+    def queue_depth
+      sql = <<~SQL
+        SELECT COUNT(*) FROM #{ready_executions_table}
+        WHERE 1=1
+        #{queue_filter_clause}
+      SQL
+      connection.select_value(sql).to_i
+    end
+
+    def oldest_job_age_seconds
+      sql = <<~SQL
+        SELECT EXTRACT(EPOCH FROM (NOW() - MIN(created_at)))
+        FROM #{ready_executions_table}
+        WHERE 1=1
+        #{queue_filter_clause}
+      SQL
+      result = connection.select_value(sql)
+      result.to_f
+    end
+
+    def jobs_per_minute
+      sql = <<~SQL
+        SELECT COUNT(*)
+        FROM #{jobs_table}
+        WHERE finished_at IS NOT NULL
+          AND finished_at > NOW() - INTERVAL '1 minute'
+          #{queue_filter_clause('queue_name')}
+      SQL
+      connection.select_value(sql).to_i
+    end
+
+    def claimed_jobs_count
+      sql = <<~SQL
+        SELECT COUNT(*) FROM #{claimed_executions_table}
+      SQL
+      connection.select_value(sql).to_i
+    end
+
+    def failed_jobs_count
+      sql = <<~SQL
+        SELECT COUNT(*) FROM #{failed_executions_table}
+      SQL
+      connection.select_value(sql).to_i
+    end
+
+    def blocked_jobs_count
+      sql = <<~SQL
+        SELECT COUNT(*) FROM #{blocked_executions_table}
+      SQL
+      connection.select_value(sql).to_i
+    end
+
+    def active_workers_count
+      sql = <<~SQL
+        SELECT COUNT(*)
+        FROM #{processes_table}
+        WHERE kind = 'Worker'
+          AND last_heartbeat_at > NOW() - INTERVAL '5 minutes'
+      SQL
+      connection.select_value(sql).to_i
+    end
+
+    def queues_breakdown
+      sql = <<~SQL
+        SELECT queue_name, COUNT(*) as count
+        FROM #{ready_executions_table}
+        GROUP BY queue_name
+        ORDER BY count DESC
+      SQL
+      connection.select_all(sql).to_a.to_h { |row| [row['queue_name'], row['count'].to_i] }
+    end
+
+    private
+
+    def connection
+      @config.connection
+    end
+
+    def queue_filter_clause(column_name = 'queue_name')
+      return '' unless @config.queues&.any?
+
+      quoted_queues = @config.queues.map { |q| connection.quote(q) }.join(', ')
+      "AND #{column_name} IN (#{quoted_queues})"
+    end
+
+    # Table name helpers using configurable prefix
+    def table_prefix
+      @config.table_prefix
+    end
+
+    def ready_executions_table
+      "#{table_prefix}ready_executions"
+    end
+
+    def jobs_table
+      "#{table_prefix}jobs"
+    end
+
+    def claimed_executions_table
+      "#{table_prefix}claimed_executions"
+    end
+
+    def failed_executions_table
+      "#{table_prefix}failed_executions"
+    end
+
+    def blocked_executions_table
+      "#{table_prefix}blocked_executions"
+    end
+
+    def processes_table
+      "#{table_prefix}processes"
+    end
+  end
+end

data/lib/solid_queue_heroku_autoscaler/railtie.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+module SolidQueueHerokuAutoscaler
+  class Railtie < Rails::Railtie
+    initializer 'solid_queue_heroku_autoscaler.configure' do
+      # Configuration happens via initializer, nothing to do here
+    end
+
+    rake_tasks do
+      namespace :solid_queue_autoscaler do
+        desc 'Run the autoscaler once for a specific worker (default: :default). Use WORKER=name'
+        task scale: :environment do
+          worker_name = (ENV['WORKER'] || 'default').to_sym
+          result = SolidQueueHerokuAutoscaler.scale!(worker_name)
+          print_scale_result(result, worker_name)
+        end
+
+        desc 'Run the autoscaler for all configured workers'
+        task scale_all: :environment do
+          results = SolidQueueHerokuAutoscaler.scale_all!
+          if results.empty?
+            puts 'No workers configured'
+            exit 1
+          end
+          results.each do |worker_name, result|
+            print_scale_result(result, worker_name)
+          end
+          exit 1 if results.values.any? { |r| !r.success? }
+        end
+
+        desc 'List all configured workers'
+        task workers: :environment do
+          workers = SolidQueueHerokuAutoscaler.registered_workers
+          if workers.empty?
+            puts 'No workers configured'
+          else
+            puts "Configured Workers (#{workers.size}):"
+            workers.each do |name|
+              config = SolidQueueHerokuAutoscaler.config(name)
+              queues = config.queues&.join(', ') || 'all'
+              puts "  #{name}:"
+              puts "    Process Type: #{config.process_type}"
+              puts "    Queues: #{queues}"
+              puts "    Workers: #{config.min_workers}-#{config.max_workers}"
+            end
+          end
+        end
+
+        desc 'Show current queue metrics for a worker. Use WORKER=name'
+        task metrics: :environment do
+          worker_name = (ENV['WORKER'] || 'default').to_sym
+          metrics = SolidQueueHerokuAutoscaler.metrics(worker_name)
+          config = SolidQueueHerokuAutoscaler.config(worker_name)
+          puts "Queue Metrics#{" [#{worker_name}]" unless worker_name == :default}:"
+          puts "  Queues Filter: #{config.queues&.join(', ') || 'all'}"
+          puts "  Queue Depth: #{metrics.queue_depth}"
+          puts "  Oldest Job Age: #{metrics.oldest_job_age_seconds.round}s"
+          puts "  Jobs/Minute: #{metrics.jobs_per_minute}"
+          puts "  Claimed Jobs: #{metrics.claimed_jobs}"
+          puts "  Failed Jobs: #{metrics.failed_jobs}"
+          puts "  Blocked Jobs: #{metrics.blocked_jobs}"
+          puts "  Active Workers: #{metrics.active_workers}"
+          puts "  Queues Breakdown: #{metrics.queues_breakdown}"
+        end
+
+        desc 'Show current worker formation. Use WORKER=name'
+        task formation: :environment do
+          worker_name = (ENV['WORKER'] || 'default').to_sym
+          workers = SolidQueueHerokuAutoscaler.current_workers(worker_name)
+          config = SolidQueueHerokuAutoscaler.config(worker_name)
+          puts "Current Formation#{" [#{worker_name}]" unless worker_name == :default}:"
+          puts "  Process Type: #{config.process_type}"
+          puts "  Workers: #{workers}"
+          puts "  Min: #{config.min_workers}"
+          puts "  Max: #{config.max_workers}"
+          puts "  Queues: #{config.queues&.join(', ') || 'all'}"
+        end
+
+        desc 'Show cooldown state for a worker. Use WORKER=name'
+        task cooldown: :environment do
+          worker_name = (ENV['WORKER'] || 'default').to_sym
+          config = SolidQueueHerokuAutoscaler.config(worker_name)
+          tracker = SolidQueueHerokuAutoscaler::CooldownTracker.new(config: config, key: worker_name.to_s)
+
+          puts "Cooldown State#{" [#{worker_name}]" unless worker_name == :default}:"
+          puts "  Table Exists: #{tracker.table_exists?}"
+
+          if tracker.table_exists?
+            state = tracker.state
+            puts "  Last Scale Up: #{state[:last_scale_up_at] || 'never'}"
+            puts "  Last Scale Down: #{state[:last_scale_down_at] || 'never'}"
+            puts "  Scale Up Cooldown Active: #{tracker.cooldown_active_for_scale_up?}"
+            puts "  Scale Down Cooldown Active: #{tracker.cooldown_active_for_scale_down?}"
+
+            if tracker.cooldown_active_for_scale_up?
+              puts "  Scale Up Cooldown Remaining: #{tracker.scale_up_cooldown_remaining.round}s"
+            end
+            if tracker.cooldown_active_for_scale_down?
+              puts "  Scale Down Cooldown Remaining: #{tracker.scale_down_cooldown_remaining.round}s"
+            end
+          else
+            puts '  (Using in-memory cooldowns - run migration for persistence)'
+            scale_up = SolidQueueHerokuAutoscaler::Scaler.last_scale_up_at(worker_name)
+            scale_down = SolidQueueHerokuAutoscaler::Scaler.last_scale_down_at(worker_name)
+            puts "  In-Memory Scale Up: #{scale_up || 'never'}"
+            puts "  In-Memory Scale Down: #{scale_down || 'never'}"
+          end
+        end
+
+        desc 'Reset cooldown state for a worker (or all if WORKER=all). Use WORKER=name'
+        task reset_cooldown: :environment do
+          worker_name = ENV['WORKER']&.to_sym
+
+          if worker_name == :all || worker_name.nil?
+            # Reset all workers
+            SolidQueueHerokuAutoscaler.registered_workers.each do |name|
+              config = SolidQueueHerokuAutoscaler.config(name)
+              tracker = SolidQueueHerokuAutoscaler::CooldownTracker.new(config: config, key: name.to_s)
+              tracker.reset! if tracker.table_exists?
+            end
+            SolidQueueHerokuAutoscaler::Scaler.reset_cooldowns!
+            puts 'All cooldown states reset'
+          else
+            config = SolidQueueHerokuAutoscaler.config(worker_name)
+            tracker = SolidQueueHerokuAutoscaler::CooldownTracker.new(config: config, key: worker_name.to_s)
+            tracker.reset! if tracker.table_exists?
+            SolidQueueHerokuAutoscaler::Scaler.reset_cooldowns!(worker_name)
+            puts "Cooldown state reset for #{worker_name}"
+          end
+        end
+
+        def print_scale_result(result, worker_name)
+          prefix = worker_name == :default ? '' : "[#{worker_name}] "
+          if result.success?
+            if result.scaled?
+              puts "#{prefix}Scaled #{result.decision.from} -> #{result.decision.to} workers"
+            elsif result.skipped?
+              puts "#{prefix}Skipped: #{result.skipped_reason}"
+            else
+              puts "#{prefix}No change needed: #{result.decision&.reason}"
+            end
+          else
+            puts "#{prefix}Error: #{result.error}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/solid_queue_heroku_autoscaler/scaler.rb

@@ -0,0 +1,227 @@
+# frozen_string_literal: true
+
+module SolidQueueHerokuAutoscaler
+  class Scaler
+    ScaleResult = Struct.new(
+      :success,
+      :decision,
+      :metrics,
+      :error,
+      :skipped_reason,
+      :executed_at,
+      keyword_init: true
+    ) do
+      def success?
+        success == true
+      end
+
+      def skipped?
+        !skipped_reason.nil?
+      end
+
+      def scaled?
+        success? && decision && !decision.no_change?
+      end
+    end
+
+    # Per-configuration cooldown tracking for multi-worker support
+    class << self
+      def cooldown_mutex
+        @cooldown_mutex ||= Mutex.new
+      end
+
+      def cooldowns
+        @cooldowns ||= {}
+      end
+
+      def last_scale_up_at(config_name = :default)
+        cooldown_mutex.synchronize { cooldowns.dig(config_name, :scale_up) }
+      end
+
+      def set_last_scale_up_at(config_name, value)
+        cooldown_mutex.synchronize do
+          cooldowns[config_name] ||= {}
+          cooldowns[config_name][:scale_up] = value
+        end
+      end
+
+      def last_scale_down_at(config_name = :default)
+        cooldown_mutex.synchronize { cooldowns.dig(config_name, :scale_down) }
+      end
+
+      def set_last_scale_down_at(config_name, value)
+        cooldown_mutex.synchronize do
+          cooldowns[config_name] ||= {}
+          cooldowns[config_name][:scale_down] = value
+        end
+      end
+
+      def reset_cooldowns!(config_name = nil)
+        cooldown_mutex.synchronize do
+          if config_name
+            cooldowns.delete(config_name)
+          else
+            @cooldowns = {}
+          end
+        end
+      end
+
+      # Backward compatibility setters
+      def last_scale_up_at=(value)
+        set_last_scale_up_at(:default, value)
+      end
+
+      def last_scale_down_at=(value)
+        set_last_scale_down_at(:default, value)
+      end
+    end
+
+    def initialize(config: nil)
+      @config = config || SolidQueueHerokuAutoscaler.config
+      @lock = AdvisoryLock.new(config: @config)
+      @metrics_collector = Metrics.new(config: @config)
+      @decision_engine = DecisionEngine.new(config: @config)
+      @adapter = @config.adapter
+    end
+
+    def run
+      return skipped_result('Autoscaler is disabled') unless @config.enabled?
+
+      return skipped_result('Could not acquire advisory lock (another instance is running)') unless @lock.try_lock
+
+      begin
+        execute_scaling
+      ensure
+        @lock.release
+      end
+    end
+
+    def run!
+      @lock.with_lock do
+        execute_scaling
+      end
+    end
+
+    private
+
+    def execute_scaling
+      metrics = @metrics_collector.collect
+      current_workers = @adapter.current_workers
+      decision = @decision_engine.decide(metrics: metrics, current_workers: current_workers)
+
+      log_decision(decision, metrics)
+
+      return success_result(decision, metrics) if decision.no_change?
+
+      if cooldown_active?(decision)
+        remaining = cooldown_remaining(decision)
+        return skipped_result("Cooldown active (#{remaining.round}s remaining)", decision: decision, metrics: metrics)
+      end
+
+      apply_decision(decision, metrics)
+    rescue StandardError => e
+      error_result(e)
+    end
+
+    def apply_decision(decision, metrics)
+      @adapter.scale(decision.to)
+      record_scale_time(decision)
+
+      log_scale_action(decision)
+
+      success_result(decision, metrics)
+    end
+
+    def cooldown_active?(decision)
+      config_name = @config.name
+      if decision.scale_up?
+        last_scale_up = self.class.last_scale_up_at(config_name)
+        return false unless last_scale_up
+
+        Time.current - last_scale_up < @config.effective_scale_up_cooldown
+      elsif decision.scale_down?
+        last_scale_down = self.class.last_scale_down_at(config_name)
+        return false unless last_scale_down
+
+        Time.current - last_scale_down < @config.effective_scale_down_cooldown
+      else
+        false
+      end
+    end
+
+    def cooldown_remaining(decision)
+      config_name = @config.name
+      if decision.scale_up?
+        elapsed = Time.current - self.class.last_scale_up_at(config_name)
+        @config.effective_scale_up_cooldown - elapsed
+      else
+        elapsed = Time.current - self.class.last_scale_down_at(config_name)
+        @config.effective_scale_down_cooldown - elapsed
+      end
+    end
+
+    def record_scale_time(decision)
+      config_name = @config.name
+      if decision.scale_up?
+        self.class.set_last_scale_up_at(config_name, Time.current)
+      elsif decision.scale_down?
+        self.class.set_last_scale_down_at(config_name, Time.current)
+      end
+    end
+
+    def log_decision(decision, metrics)
+      worker_label = @config.name == :default ? '' : "[#{@config.name}] "
+      logger.info(
+        "[Autoscaler] #{worker_label}Evaluated: action=#{decision.action} " \
+        "workers=#{decision.from}->#{decision.to} " \
+        "queue_depth=#{metrics.queue_depth} " \
+        "latency=#{metrics.oldest_job_age_seconds.round}s " \
+        "reason=\"#{decision.reason}\""
+      )
+    end
+
+    def log_scale_action(decision)
+      prefix = @config.dry_run? ? '[DRY RUN] ' : ''
+      worker_label = @config.name == :default ? '' : "[#{@config.name}] "
+      logger.info(
+        "#{prefix}[Autoscaler] #{worker_label}Scaling #{decision.action}: " \
+        "#{decision.from} -> #{decision.to} workers (#{decision.reason})"
+      )
+    end
+
+    def success_result(decision, metrics)
+      ScaleResult.new(
+        success: true,
+        decision: decision,
+        metrics: metrics,
+        executed_at: Time.current
+      )
+    end
+
+    def skipped_result(reason, decision: nil, metrics: nil)
+      logger.debug("[Autoscaler] Skipped: #{reason}")
+
+      ScaleResult.new(
+        success: true,
+        decision: decision,
+        metrics: metrics,
+        skipped_reason: reason,
+        executed_at: Time.current
+      )
+    end
+
+    def error_result(error)
+      logger.error("[Autoscaler] Error: #{error.class}: #{error.message}")
+
+      ScaleResult.new(
+        success: false,
+        error: error,
+        executed_at: Time.current
+      )
+    end
+
+    def logger
+      @config.logger
+    end
+  end
+end

data/lib/solid_queue_heroku_autoscaler/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module SolidQueueHerokuAutoscaler
+  VERSION = '0.1.0'
+end

data/lib/solid_queue_heroku_autoscaler.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+require 'active_record'
+require 'active_support'
+require 'active_support/core_ext/numeric/time'
+
+require_relative 'solid_queue_heroku_autoscaler/version'
+require_relative 'solid_queue_heroku_autoscaler/errors'
+require_relative 'solid_queue_heroku_autoscaler/adapters'
+require_relative 'solid_queue_heroku_autoscaler/configuration'
+require_relative 'solid_queue_heroku_autoscaler/advisory_lock'
+require_relative 'solid_queue_heroku_autoscaler/metrics'
+require_relative 'solid_queue_heroku_autoscaler/decision_engine'
+require_relative 'solid_queue_heroku_autoscaler/cooldown_tracker'
+require_relative 'solid_queue_heroku_autoscaler/scaler'
+
+module SolidQueueHerokuAutoscaler
+  class << self
+    # Registry of named configurations for multi-worker support
+    def configurations
+      @configurations ||= {}
+    end
+
+    # Configure a named worker type (default: :default for backward compatibility)
+    # @param name [Symbol] The name of the worker type (e.g., :critical_worker, :default_worker)
+    # @yield [Configuration] The configuration object to customize
+    # @return [Configuration] The configured configuration object
+    def configure(name = :default)
+      config_obj = configurations[name] ||= Configuration.new
+      config_obj.name = name
+      yield(config_obj) if block_given?
+      config_obj.validate!
+      config_obj
+    end
+
+    # Get configuration for a named worker type
+    # @param name [Symbol] The name of the worker type
+    # @return [Configuration] The configuration object
+    def config(name = :default)
+      configurations[name] || configure(name)
+    end
+
+    # Scale a specific worker type
+    # @param name [Symbol] The name of the worker type to scale
+    # @return [Scaler::ScaleResult] The result of the scaling operation
+    def scale!(name = :default)
+      Scaler.new(config: config(name)).run
+    end
+
+    # Scale all configured worker types
+    # @return [Hash<Symbol, Scaler::ScaleResult>] Results keyed by worker name
+    def scale_all!
+      return {} if configurations.empty?
+
+      # Copy keys to avoid modifying hash during iteration
+      worker_names = configurations.keys.dup
+      worker_names.each_with_object({}) do |name, results|
+        results[name] = scale!(name)
+      end
+    end
+
+    # Get metrics for a specific worker type
+    # @param name [Symbol] The name of the worker type
+    # @return [Metrics::Result] The collected metrics
+    def metrics(name = :default)
+      Metrics.new(config: config(name)).collect
+    end
+
+    # Get current worker count for a specific worker type
+    # @param name [Symbol] The name of the worker type
+    # @return [Integer] The current number of workers
+    def current_workers(name = :default)
+      config(name).adapter.current_workers
+    end
+
+    # List all registered worker type names
+    # @return [Array<Symbol>] List of configured worker names
+    def registered_workers
+      configurations.keys
+    end
+
+    # Reset all configurations (useful for testing)
+    def reset_configuration!
+      @configurations = {}
+      Scaler.reset_cooldowns!
+    end
+
+    # Backward compatibility: single configuration accessor
+    def configuration
+      configurations[:default]
+    end
+
+    def configuration=(config_obj)
+      if config_obj.nil?
+        @configurations = {}
+      else
+        config_obj.name ||= :default
+        configurations[:default] = config_obj
+      end
+    end
+  end
+end
+
+require_relative 'solid_queue_heroku_autoscaler/railtie' if defined?(Rails::Railtie)
+
+require_relative 'solid_queue_heroku_autoscaler/autoscale_job' if defined?(ActiveJob::Base)