solid_queue_heroku_autoscaler 0.1.0

data/lib/solid_queue_heroku_autoscaler/configuration.rb

@@ -0,0 +1,217 @@
+# frozen_string_literal: true
+
+require 'logger'
+
+module SolidQueueHerokuAutoscaler
+  class Configuration
+    # Configuration name (for multi-worker support)
+    attr_accessor :name
+
+    # Heroku settings
+    attr_accessor :heroku_api_key
+
+    # Worker limits
+    attr_accessor :min_workers
+
+    # Scale-up thresholds
+    attr_accessor :scale_up_queue_depth
+
+    # Scale-down thresholds
+    attr_accessor :scale_down_queue_depth
+
+    # Scaling strategy
+    attr_accessor :scaling_strategy
+
+    # Safety settings
+    attr_accessor :cooldown_seconds
+
+    # Advisory lock settings
+    attr_accessor :lock_timeout_seconds
+
+    # Behavior settings
+    attr_accessor :dry_run
+
+    # Queue filtering
+    attr_accessor :queues
+
+    # Database connection
+    attr_accessor :database_connection
+
+    # Solid Queue table prefix (default: 'solid_queue_')
+    attr_accessor :table_prefix
+
+    # Infrastructure adapter (defaults to Heroku)
+    attr_accessor :adapter_class
+
+    # Kubernetes settings (for Kubernetes adapter)
+    attr_accessor :kubernetes_deployment, :kubernetes_namespace, :kubernetes_context, :kubernetes_kubeconfig
+
+    # Additional Heroku settings
+    attr_accessor :heroku_app_name, :process_type, :max_workers
+
+    # Scale-up settings
+    attr_accessor :scale_up_latency_seconds, :scale_up_increment
+
+    # Scale-down settings
+    attr_accessor :scale_down_latency_seconds, :scale_down_idle_minutes, :scale_down_decrement
+    attr_accessor :scale_up_jobs_per_worker, :scale_up_latency_per_worker, :scale_up_cooldown_seconds, :scale_down_jobs_per_worker, :scale_down_cooldown_seconds
+
+    # Other settings
+    attr_accessor :enabled, :logger
+    attr_writer :lock_key
+
+    def initialize
+      # Configuration name (auto-set when using named configurations)
+      @name = :default
+
+      # Heroku settings - required
+      @heroku_api_key = ENV.fetch('HEROKU_API_KEY', nil)
+      @heroku_app_name = ENV.fetch('HEROKU_APP_NAME', nil)
+      @process_type = 'worker'
+
+      # Worker limits
+      @min_workers = 1
+      @max_workers = 10
+
+      # Scale-up thresholds
+      @scale_up_queue_depth = 100
+      @scale_up_latency_seconds = 300
+      @scale_up_increment = 1
+
+      # Scale-down thresholds
+      @scale_down_queue_depth = 10
+      @scale_down_latency_seconds = 30
+      @scale_down_idle_minutes = 5
+      @scale_down_decrement = 1
+
+      # Scaling strategy (:fixed or :proportional)
+      @scaling_strategy = :fixed
+      @scale_up_jobs_per_worker = 50
+      @scale_up_latency_per_worker = 60
+      @scale_down_jobs_per_worker = 50
+
+      # Safety settings
+      @cooldown_seconds = 120
+      @scale_up_cooldown_seconds = nil
+      @scale_down_cooldown_seconds = nil
+
+      # Advisory lock settings
+      @lock_timeout_seconds = 30
+      @lock_key = nil # Auto-generated based on name if not set
+
+      # Behavior
+      @dry_run = false
+      @enabled = true
+      @logger = default_logger
+
+      # Queue filtering (nil = all queues)
+      @queues = nil
+
+      # Database connection (defaults to ActiveRecord::Base.connection)
+      @database_connection = nil
+
+      # Solid Queue table prefix (default: 'solid_queue_')
+      @table_prefix = 'solid_queue_'
+
+      # Infrastructure adapter (defaults to Heroku)
+      @adapter_class = nil
+
+      # Kubernetes settings (for Kubernetes adapter)
+      @kubernetes_deployment = ENV.fetch('K8S_DEPLOYMENT', nil)
+      @kubernetes_namespace = ENV['K8S_NAMESPACE'] || 'default'
+      @kubernetes_context = ENV.fetch('K8S_CONTEXT', nil)
+      @kubernetes_kubeconfig = ENV.fetch('KUBECONFIG', nil)
+    end
+
+    # Returns the lock key, auto-generating based on name if not explicitly set
+    # Each worker type gets a unique lock to allow parallel scaling
+    def lock_key
+      @lock_key || "solid_queue_autoscaler_#{name}"
+    end
+
+    VALID_SCALING_STRATEGIES = %i[fixed proportional].freeze
+
+    def validate!
+      errors = []
+
+      # Validate adapter-specific configuration
+      errors.concat(adapter.configuration_errors)
+
+      errors << 'min_workers must be >= 0' if min_workers.negative?
+      errors << 'max_workers must be > 0' if max_workers <= 0
+      errors << 'min_workers cannot exceed max_workers' if min_workers > max_workers
+
+      errors << 'scale_up_queue_depth must be > 0' if scale_up_queue_depth <= 0
+      errors << 'scale_up_latency_seconds must be > 0' if scale_up_latency_seconds <= 0
+      errors << 'scale_up_increment must be > 0' if scale_up_increment <= 0
+
+      errors << 'scale_down_queue_depth must be >= 0' if scale_down_queue_depth.negative?
+      errors << 'scale_down_decrement must be > 0' if scale_down_decrement <= 0
+
+      errors << 'cooldown_seconds must be >= 0' if cooldown_seconds.negative?
+      errors << 'lock_timeout_seconds must be > 0' if lock_timeout_seconds <= 0
+
+      if table_prefix.nil? || table_prefix.to_s.strip.empty?
+        errors << 'table_prefix cannot be nil or empty'
+      elsif !table_prefix.to_s.end_with?('_')
+        errors << 'table_prefix must end with an underscore'
+      end
+
+      unless VALID_SCALING_STRATEGIES.include?(scaling_strategy)
+        errors << "scaling_strategy must be one of: #{VALID_SCALING_STRATEGIES.join(', ')}"
+      end
+
+      raise ConfigurationError, errors.join(', ') if errors.any?
+
+      true
+    end
+
+    def effective_scale_up_cooldown
+      scale_up_cooldown_seconds || cooldown_seconds
+    end
+
+    def effective_scale_down_cooldown
+      scale_down_cooldown_seconds || cooldown_seconds
+    end
+
+    def connection
+      database_connection || ActiveRecord::Base.connection
+    end
+
+    def dry_run?
+      dry_run
+    end
+
+    def enabled?
+      enabled
+    end
+
+    # Returns the configured adapter instance.
+    # Creates a new instance from adapter_class if not set.
+    # Defaults to Heroku adapter.
+    def adapter
+      @adapter ||= begin
+        klass = adapter_class || Adapters::Heroku
+        klass.new(config: self)
+      end
+    end
+
+    # Allow setting a pre-configured adapter instance
+    attr_writer :adapter
+
+    private
+
+    def default_logger
+      if defined?(Rails) && Rails.logger
+        Rails.logger
+      else
+        Logger.new($stdout).tap do |logger|
+          logger.level = Logger::INFO
+          logger.formatter = proc do |severity, datetime, _progname, msg|
+            "[#{datetime.strftime('%Y-%m-%d %H:%M:%S')}] [SolidQueueAutoscaler] #{severity}: #{msg}\n"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/solid_queue_heroku_autoscaler/cooldown_tracker.rb

@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+
+require 'time'
+
+module SolidQueueHerokuAutoscaler
+  class CooldownTracker
+    TABLE_NAME = 'solid_queue_autoscaler_state'
+    DEFAULT_KEY = 'default'
+
+    attr_reader :key
+
+    def initialize(config: nil, key: DEFAULT_KEY)
+      @config = config || SolidQueueHerokuAutoscaler.config
+      @key = key
+      @table_exists = nil
+    end
+
+    def last_scale_up_at
+      return nil unless table_exists?
+
+      result = connection.select_value(<<~SQL)
+        SELECT last_scale_up_at FROM #{TABLE_NAME}
+        WHERE key = #{connection.quote(key)}
+      SQL
+      result ? Time.parse(result.to_s) : nil
+    rescue ArgumentError
+      nil
+    end
+
+    def last_scale_down_at
+      return nil unless table_exists?
+
+      result = connection.select_value(<<~SQL)
+        SELECT last_scale_down_at FROM #{TABLE_NAME}
+        WHERE key = #{connection.quote(key)}
+      SQL
+      result ? Time.parse(result.to_s) : nil
+    rescue ArgumentError
+      nil
+    end
+
+    def record_scale_up!
+      return false unless table_exists?
+
+      upsert_state(last_scale_up_at: Time.current)
+      true
+    end
+
+    def record_scale_down!
+      return false unless table_exists?
+
+      upsert_state(last_scale_down_at: Time.current)
+      true
+    end
+
+    def reset!
+      return false unless table_exists?
+
+      connection.execute(<<~SQL)
+        DELETE FROM #{TABLE_NAME} WHERE key = #{connection.quote(key)}
+      SQL
+      true
+    end
+
+    def cooldown_active_for_scale_up?
+      last = last_scale_up_at
+      return false unless last
+
+      Time.current - last < @config.effective_scale_up_cooldown
+    end
+
+    def cooldown_active_for_scale_down?
+      last = last_scale_down_at
+      return false unless last
+
+      Time.current - last < @config.effective_scale_down_cooldown
+    end
+
+    def scale_up_cooldown_remaining
+      last = last_scale_up_at
+      return 0 unless last
+
+      remaining = @config.effective_scale_up_cooldown - (Time.current - last)
+      [remaining, 0].max
+    end
+
+    def scale_down_cooldown_remaining
+      last = last_scale_down_at
+      return 0 unless last
+
+      remaining = @config.effective_scale_down_cooldown - (Time.current - last)
+      [remaining, 0].max
+    end
+
+    def table_exists?
+      return @table_exists unless @table_exists.nil?
+
+      @table_exists = connection.table_exists?(TABLE_NAME)
+    rescue StandardError
+      @table_exists = false
+    end
+
+    def state
+      return {} unless table_exists?
+
+      row = connection.select_one(<<~SQL)
+        SELECT last_scale_up_at, last_scale_down_at, updated_at
+        FROM #{TABLE_NAME}
+        WHERE key = #{connection.quote(key)}
+      SQL
+
+      return {} unless row
+
+      {
+        last_scale_up_at: row['last_scale_up_at'],
+        last_scale_down_at: row['last_scale_down_at'],
+        updated_at: row['updated_at']
+      }
+    end
+
+    private
+
+    def connection
+      @config.connection
+    end
+
+    def upsert_state(last_scale_up_at: nil, last_scale_down_at: nil)
+      now = Time.current
+      quoted_key = connection.quote(key)
+      quoted_now = connection.quote(now)
+
+      if last_scale_up_at
+        quoted_time = connection.quote(last_scale_up_at)
+        connection.execute(<<~SQL)
+          INSERT INTO #{TABLE_NAME} (key, last_scale_up_at, created_at, updated_at)
+          VALUES (#{quoted_key}, #{quoted_time}, #{quoted_now}, #{quoted_now})
+          ON CONFLICT (key) DO UPDATE SET
+            last_scale_up_at = EXCLUDED.last_scale_up_at,
+            updated_at = EXCLUDED.updated_at
+        SQL
+      elsif last_scale_down_at
+        quoted_time = connection.quote(last_scale_down_at)
+        connection.execute(<<~SQL)
+          INSERT INTO #{TABLE_NAME} (key, last_scale_down_at, created_at, updated_at)
+          VALUES (#{quoted_key}, #{quoted_time}, #{quoted_now}, #{quoted_now})
+          ON CONFLICT (key) DO UPDATE SET
+            last_scale_down_at = EXCLUDED.last_scale_down_at,
+            updated_at = EXCLUDED.updated_at
+        SQL
+      end
+    end
+  end
+end

data/lib/solid_queue_heroku_autoscaler/decision_engine.rb

@@ -0,0 +1,228 @@
+# frozen_string_literal: true
+
+module SolidQueueHerokuAutoscaler
+  class DecisionEngine
+    Decision = Struct.new(:action, :from, :to, :reason, keyword_init: true) do
+      def scale_up?
+        action == :scale_up
+      end
+
+      def scale_down?
+        action == :scale_down
+      end
+
+      def no_change?
+        action == :no_change
+      end
+
+      def delta
+        to - from
+      end
+    end
+
+    def initialize(config: nil)
+      @config = config || SolidQueueHerokuAutoscaler.config
+    end
+
+    def decide(metrics:, current_workers:)
+      return no_change_decision(current_workers, 'Autoscaler is disabled') unless @config.enabled?
+
+      if should_scale_up?(metrics, current_workers)
+        scale_up_decision(metrics, current_workers)
+      elsif should_scale_down?(metrics, current_workers)
+        scale_down_decision(metrics, current_workers)
+      else
+        no_change_decision(current_workers, determine_no_change_reason(metrics, current_workers))
+      end
+    end
+
+    private
+
+    def should_scale_up?(metrics, current_workers)
+      return false if current_workers >= @config.max_workers
+
+      queue_depth_high = metrics.queue_depth >= @config.scale_up_queue_depth
+      latency_high = metrics.oldest_job_age_seconds >= @config.scale_up_latency_seconds
+
+      queue_depth_high || latency_high
+    end
+
+    def should_scale_down?(metrics, current_workers)
+      return false if current_workers <= @config.min_workers
+
+      queue_depth_low = metrics.queue_depth <= @config.scale_down_queue_depth
+      latency_low = metrics.oldest_job_age_seconds <= @config.scale_down_latency_seconds
+      is_idle = metrics.idle?
+
+      (queue_depth_low && latency_low) || is_idle
+    end
+
+    def scale_up_decision(metrics, current_workers)
+      target = calculate_scale_up_target(metrics, current_workers)
+      reason = build_scale_up_reason(metrics, current_workers, target)
+
+      Decision.new(
+        action: :scale_up,
+        from: current_workers,
+        to: target,
+        reason: reason
+      )
+    end
+
+    def scale_down_decision(metrics, current_workers)
+      target = calculate_scale_down_target(metrics, current_workers)
+      reason = build_scale_down_reason(metrics, current_workers, target)
+
+      Decision.new(
+        action: :scale_down,
+        from: current_workers,
+        to: target,
+        reason: reason
+      )
+    end
+
+    def calculate_scale_up_target(metrics, current_workers)
+      raw_target = case @config.scaling_strategy
+                   when :proportional
+                     calculate_proportional_scale_up_target(metrics, current_workers)
+                   when :step_function
+                     calculate_step_function_target(metrics, current_workers)
+                   else # :fixed
+                     current_workers + @config.scale_up_increment
+                   end
+
+      [raw_target, @config.max_workers].min
+    end
+
+    def calculate_scale_down_target(metrics, current_workers)
+      raw_target = case @config.scaling_strategy
+                   when :proportional
+                     calculate_proportional_scale_down_target(metrics, current_workers)
+                   when :step_function
+                     calculate_step_function_target(metrics, current_workers)
+                   else # :fixed
+                     current_workers - @config.scale_down_decrement
+                   end
+
+      [raw_target, @config.min_workers].max
+    end
+
+    def calculate_proportional_scale_up_target(metrics, current_workers)
+      # Calculate workers needed based on queue depth
+      jobs_over_threshold = [metrics.queue_depth - @config.scale_up_queue_depth, 0].max
+      workers_for_depth = (jobs_over_threshold.to_f / @config.scale_up_jobs_per_worker).ceil
+
+      # Calculate workers needed based on latency
+      latency_over_threshold = [metrics.oldest_job_age_seconds - @config.scale_up_latency_seconds, 0].max
+      workers_for_latency = (latency_over_threshold / @config.scale_up_latency_per_worker).ceil
+
+      # Take the higher of the two calculations
+      additional_workers = [workers_for_depth, workers_for_latency].max
+
+      # Always add at least scale_up_increment if we're scaling up
+      additional_workers = [@config.scale_up_increment, additional_workers].max
+
+      current_workers + additional_workers
+    end
+
+    def calculate_proportional_scale_down_target(metrics, current_workers)
+      # If idle, scale down aggressively
+      return @config.min_workers if metrics.idle?
+
+      # Calculate how much capacity we have based on queue depth
+      jobs_under_capacity = [@config.scale_down_queue_depth - metrics.queue_depth, 0].max
+      workers_to_remove = (jobs_under_capacity.to_f / @config.scale_down_jobs_per_worker).floor
+
+      # Ensure we remove at least scale_down_decrement if we're scaling down
+      workers_to_remove = [@config.scale_down_decrement, workers_to_remove].max
+
+      current_workers - workers_to_remove
+    end
+
+    def calculate_step_function_target(metrics, current_workers)
+      # Step function uses fixed thresholds (future implementation)
+      # For now, fall back to fixed strategy
+      if should_scale_up?(metrics, current_workers)
+        current_workers + @config.scale_up_increment
+      else
+        current_workers - @config.scale_down_decrement
+      end
+    end
+
+    def no_change_decision(current_workers, reason)
+      Decision.new(
+        action: :no_change,
+        from: current_workers,
+        to: current_workers,
+        reason: reason
+      )
+    end
+
+    def build_scale_up_reason(metrics, current_workers = nil, target = nil)
+      reasons = []
+
+      if metrics.queue_depth >= @config.scale_up_queue_depth
+        reasons << "queue_depth=#{metrics.queue_depth} >= #{@config.scale_up_queue_depth}"
+      end
+
+      if metrics.oldest_job_age_seconds >= @config.scale_up_latency_seconds
+        reasons << "latency=#{metrics.oldest_job_age_seconds.round}s >= #{@config.scale_up_latency_seconds}s"
+      end
+
+      base_reason = reasons.join(', ')
+
+      if @config.scaling_strategy == :proportional && current_workers && target
+        delta = target - current_workers
+        "#{base_reason} [proportional: +#{delta} workers]"
+      else
+        base_reason
+      end
+    end
+
+    def build_scale_down_reason(metrics, current_workers = nil, target = nil)
+      if metrics.idle?
+        base_reason = 'queue is idle (no pending or claimed jobs)'
+      else
+        reasons = []
+
+        if metrics.queue_depth <= @config.scale_down_queue_depth
+          reasons << "queue_depth=#{metrics.queue_depth} <= #{@config.scale_down_queue_depth}"
+        end
+
+        if metrics.oldest_job_age_seconds <= @config.scale_down_latency_seconds
+          reasons << "latency=#{metrics.oldest_job_age_seconds.round}s <= #{@config.scale_down_latency_seconds}s"
+        end
+
+        base_reason = reasons.join(', ')
+      end
+
+      if @config.scaling_strategy == :proportional && current_workers && target
+        delta = current_workers - target
+        "#{base_reason} [proportional: -#{delta} workers]"
+      else
+        base_reason
+      end
+    end
+
+    def determine_no_change_reason(metrics, current_workers)
+      # Check if we would scale up but we're at max
+      queue_depth_high = metrics.queue_depth >= @config.scale_up_queue_depth
+      latency_high = metrics.oldest_job_age_seconds >= @config.scale_up_latency_seconds
+      would_scale_up = queue_depth_high || latency_high
+
+      # Check if we would scale down but we're at min
+      queue_depth_low = metrics.queue_depth <= @config.scale_down_queue_depth
+      latency_low = metrics.oldest_job_age_seconds <= @config.scale_down_latency_seconds
+      is_idle = metrics.idle?
+      would_scale_down = (queue_depth_low && latency_low) || is_idle
+
+      if current_workers >= @config.max_workers && would_scale_up
+        "at max_workers (#{@config.max_workers})"
+      elsif current_workers <= @config.min_workers && would_scale_down
+        "at min_workers (#{@config.min_workers})"
+      else
+        "metrics within normal range (depth=#{metrics.queue_depth}, latency=#{metrics.oldest_job_age_seconds.round}s)"
+      end
+    end
+  end
+end

data/lib/solid_queue_heroku_autoscaler/errors.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module SolidQueueHerokuAutoscaler
+  # Base error class for all autoscaler errors.
+  class Error < StandardError; end
+
+  # Raised when configuration is invalid.
+  class ConfigurationError < Error; end
+
+  # Raised when the advisory lock cannot be acquired.
+  class LockError < Error; end
+
+  # Raised when Heroku API calls fail.
+  class HerokuAPIError < Error
+    attr_reader :status_code, :response_body
+
+    def initialize(message, status_code: nil, response_body: nil)
+      super(message)
+      @status_code = status_code
+      @response_body = response_body
+    end
+  end
+
+  # Raised when Kubernetes API calls fail.
+  class KubernetesAPIError < Error
+    attr_reader :original_error
+
+    def initialize(message, original_error: nil)
+      super(message)
+      @original_error = original_error
+    end
+  end
+
+  class MetricsError < Error; end
+
+  class CooldownActiveError < Error
+    attr_reader :remaining_seconds
+
+    def initialize(remaining_seconds)
+      @remaining_seconds = remaining_seconds
+      super("Cooldown active, #{remaining_seconds.round}s remaining")
+    end
+  end
+end