solid_queue_autoscaler 1.0.0

data/lib/solid_queue_autoscaler/configuration.rb

@@ -0,0 +1,269 @@
+# frozen_string_literal: true
+
+require 'logger'
+
+module SolidQueueAutoscaler
+  class Configuration
+    # Configuration name (for multi-worker support)
+    attr_accessor :name
+
+    # Heroku settings
+    attr_accessor :heroku_api_key
+
+    # Worker limits
+    attr_accessor :min_workers
+
+    # Scale-up thresholds
+    attr_accessor :scale_up_queue_depth
+
+    # Scale-down thresholds
+    attr_accessor :scale_down_queue_depth
+
+    # Scaling strategy
+    attr_accessor :scaling_strategy
+
+    # Safety settings
+    attr_accessor :cooldown_seconds
+
+    # Advisory lock settings
+    attr_accessor :lock_timeout_seconds
+
+    # Behavior settings
+    attr_accessor :dry_run
+
+    # Queue filtering
+    attr_accessor :queues
+
+    # Database connection
+    attr_accessor :database_connection
+
+    # Solid Queue table prefix (default: 'solid_queue_')
+    attr_accessor :table_prefix
+
+    # Infrastructure adapter (defaults to Heroku)
+    attr_accessor :adapter_class
+
+    # Kubernetes settings (for Kubernetes adapter)
+    attr_accessor :kubernetes_deployment, :kubernetes_namespace, :kubernetes_context, :kubernetes_kubeconfig
+
+    # Additional Heroku settings
+    attr_accessor :heroku_app_name, :process_type, :max_workers
+
+    # Scale-up settings
+    attr_accessor :scale_up_latency_seconds, :scale_up_increment
+
+    # Scale-down settings
+    attr_accessor :scale_down_latency_seconds, :scale_down_idle_minutes, :scale_down_decrement
+    attr_accessor :scale_up_jobs_per_worker, :scale_up_latency_per_worker, :scale_up_cooldown_seconds, :scale_down_jobs_per_worker, :scale_down_cooldown_seconds
+
+    # Other settings
+    attr_accessor :enabled, :logger
+    attr_writer :lock_key
+
+    # Dashboard/event recording settings
+    attr_accessor :record_events, :record_all_events
+
+    def initialize
+      # Configuration name (auto-set when using named configurations)
+      @name = :default
+
+      # Heroku settings - required
+      @heroku_api_key = ENV.fetch('HEROKU_API_KEY', nil)
+      @heroku_app_name = ENV.fetch('HEROKU_APP_NAME', nil)
+      @process_type = 'worker'
+
+      # Worker limits
+      @min_workers = 1
+      @max_workers = 10
+
+      # Scale-up thresholds
+      @scale_up_queue_depth = 100
+      @scale_up_latency_seconds = 300
+      @scale_up_increment = 1
+
+      # Scale-down thresholds
+      @scale_down_queue_depth = 10
+      @scale_down_latency_seconds = 30
+      @scale_down_idle_minutes = 5
+      @scale_down_decrement = 1
+
+      # Scaling strategy (:fixed or :proportional)
+      @scaling_strategy = :fixed
+      @scale_up_jobs_per_worker = 50
+      @scale_up_latency_per_worker = 60
+      @scale_down_jobs_per_worker = 50
+
+      # Safety settings
+      @cooldown_seconds = 120
+      @scale_up_cooldown_seconds = nil
+      @scale_down_cooldown_seconds = nil
+
+      # Advisory lock settings
+      @lock_timeout_seconds = 30
+      @lock_key = nil # Auto-generated based on name if not set
+
+      # Behavior
+      @dry_run = false
+      @enabled = true
+      @logger = default_logger
+
+      # Queue filtering (nil = all queues)
+      @queues = nil
+
+      # Database connection (defaults to ActiveRecord::Base.connection)
+      @database_connection = nil
+
+      # Solid Queue table prefix (default: 'solid_queue_')
+      @table_prefix = 'solid_queue_'
+
+      # Infrastructure adapter (defaults to Heroku)
+      @adapter_class = nil
+
+      # Kubernetes settings (for Kubernetes adapter)
+      @kubernetes_deployment = ENV.fetch('K8S_DEPLOYMENT', nil)
+      @kubernetes_namespace = ENV['K8S_NAMESPACE'] || 'default'
+      @kubernetes_context = ENV.fetch('K8S_CONTEXT', nil)
+      @kubernetes_kubeconfig = ENV.fetch('KUBECONFIG', nil)
+
+      # Dashboard/event recording settings
+      @record_events = true # Record scale events to database
+      @record_all_events = false # Also record no_change events (verbose)
+    end
+
+    # Returns the lock key, auto-generating based on name if not explicitly set
+    # Each worker type gets a unique lock to allow parallel scaling
+    def lock_key
+      @lock_key || "solid_queue_autoscaler_#{name}"
+    end
+
+    VALID_SCALING_STRATEGIES = %i[fixed proportional].freeze
+
+    def validate!
+      errors = []
+
+      # Validate adapter-specific configuration
+      errors.concat(adapter.configuration_errors)
+
+      errors << 'min_workers must be >= 0' if min_workers.negative?
+      errors << 'max_workers must be > 0' if max_workers <= 0
+      errors << 'min_workers cannot exceed max_workers' if min_workers > max_workers
+
+      errors << 'scale_up_queue_depth must be > 0' if scale_up_queue_depth <= 0
+      errors << 'scale_up_latency_seconds must be > 0' if scale_up_latency_seconds <= 0
+      errors << 'scale_up_increment must be > 0' if scale_up_increment <= 0
+
+      errors << 'scale_down_queue_depth must be >= 0' if scale_down_queue_depth.negative?
+      errors << 'scale_down_decrement must be > 0' if scale_down_decrement <= 0
+
+      errors << 'cooldown_seconds must be >= 0' if cooldown_seconds.negative?
+      errors << 'lock_timeout_seconds must be > 0' if lock_timeout_seconds <= 0
+
+      if table_prefix.nil? || table_prefix.to_s.strip.empty?
+        errors << 'table_prefix cannot be nil or empty'
+      elsif !table_prefix.to_s.end_with?('_')
+        errors << 'table_prefix must end with an underscore'
+      end
+
+      unless VALID_SCALING_STRATEGIES.include?(scaling_strategy)
+        errors << "scaling_strategy must be one of: #{VALID_SCALING_STRATEGIES.join(', ')}"
+      end
+
+      raise ConfigurationError, errors.join(', ') if errors.any?
+
+      true
+    end
+
+    def effective_scale_up_cooldown
+      scale_up_cooldown_seconds || cooldown_seconds
+    end
+
+    def effective_scale_down_cooldown
+      scale_down_cooldown_seconds || cooldown_seconds
+    end
+
+    def connection
+      database_connection || ActiveRecord::Base.connection
+    end
+
+    def dry_run?
+      dry_run
+    end
+
+    def enabled?
+      enabled
+    end
+
+    def record_events?
+      record_events && connection_available?
+    end
+
+    def record_all_events?
+      record_all_events && record_events?
+    end
+
+    def connection_available?
+      return true if database_connection
+      return false unless defined?(ActiveRecord::Base)
+
+      ActiveRecord::Base.connected?
+    rescue StandardError
+      false
+    end
+
+    # Returns the configured adapter instance.
+    # Creates a new instance from adapter_class if not set.
+    # Defaults to Heroku adapter.
+    def adapter
+      @adapter ||= begin
+        klass = adapter_class || Adapters::Heroku
+        klass.new(config: self)
+      end
+    end
+
+    # Allow setting a pre-configured adapter instance or a symbol shortcut
+    # @param value [Symbol, Base, Class] :heroku, :kubernetes, an adapter instance, or adapter class
+    def adapter=(value)
+      @adapter = case value
+                 when Symbol
+                   resolve_adapter_symbol(value)
+                 when Class
+                   value.new(config: self)
+                 else
+                   value
+                 end
+    end
+
+    # Maps adapter symbols to adapter classes
+    ADAPTER_SYMBOLS = {
+      heroku: 'SolidQueueAutoscaler::Adapters::Heroku',
+      kubernetes: 'SolidQueueAutoscaler::Adapters::Kubernetes',
+      k8s: 'SolidQueueAutoscaler::Adapters::Kubernetes'
+    }.freeze
+
+    private
+
+    def resolve_adapter_symbol(symbol)
+      class_name = ADAPTER_SYMBOLS[symbol]
+      unless class_name
+        raise ConfigurationError,
+              "Unknown adapter: #{symbol}. Valid options: #{ADAPTER_SYMBOLS.keys.join(', ')}"
+      end
+
+      klass = class_name.split('::').reduce(Object) { |mod, name| mod.const_get(name) }
+      klass.new(config: self)
+    end
+
+    def default_logger
+      if defined?(Rails) && Rails.logger
+        Rails.logger
+      else
+        Logger.new($stdout).tap do |logger|
+          logger.level = Logger::INFO
+          logger.formatter = proc do |severity, datetime, _progname, msg|
+            "[#{datetime.strftime('%Y-%m-%d %H:%M:%S')}] [SolidQueueAutoscaler] #{severity}: #{msg}\n"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/solid_queue_autoscaler/cooldown_tracker.rb

@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+
+require 'time'
+
+module SolidQueueAutoscaler
+  class CooldownTracker
+    TABLE_NAME = 'solid_queue_autoscaler_state'
+    DEFAULT_KEY = 'default'
+
+    attr_reader :key
+
+    def initialize(config: nil, key: DEFAULT_KEY)
+      @config = config || SolidQueueAutoscaler.config
+      @key = key
+      @table_exists = nil
+    end
+
+    def last_scale_up_at
+      return nil unless table_exists?
+
+      result = connection.select_value(<<~SQL)
+        SELECT last_scale_up_at FROM #{TABLE_NAME}
+        WHERE key = #{connection.quote(key)}
+      SQL
+      result ? Time.parse(result.to_s) : nil
+    rescue ArgumentError
+      nil
+    end
+
+    def last_scale_down_at
+      return nil unless table_exists?
+
+      result = connection.select_value(<<~SQL)
+        SELECT last_scale_down_at FROM #{TABLE_NAME}
+        WHERE key = #{connection.quote(key)}
+      SQL
+      result ? Time.parse(result.to_s) : nil
+    rescue ArgumentError
+      nil
+    end
+
+    def record_scale_up!
+      return false unless table_exists?
+
+      upsert_state(last_scale_up_at: Time.current)
+      true
+    end
+
+    def record_scale_down!
+      return false unless table_exists?
+
+      upsert_state(last_scale_down_at: Time.current)
+      true
+    end
+
+    def reset!
+      return false unless table_exists?
+
+      connection.execute(<<~SQL)
+        DELETE FROM #{TABLE_NAME} WHERE key = #{connection.quote(key)}
+      SQL
+      true
+    end
+
+    def cooldown_active_for_scale_up?
+      last = last_scale_up_at
+      return false unless last
+
+      Time.current - last < @config.effective_scale_up_cooldown
+    end
+
+    def cooldown_active_for_scale_down?
+      last = last_scale_down_at
+      return false unless last
+
+      Time.current - last < @config.effective_scale_down_cooldown
+    end
+
+    def scale_up_cooldown_remaining
+      last = last_scale_up_at
+      return 0 unless last
+
+      remaining = @config.effective_scale_up_cooldown - (Time.current - last)
+      [remaining, 0].max
+    end
+
+    def scale_down_cooldown_remaining
+      last = last_scale_down_at
+      return 0 unless last
+
+      remaining = @config.effective_scale_down_cooldown - (Time.current - last)
+      [remaining, 0].max
+    end
+
+    def table_exists?
+      return @table_exists unless @table_exists.nil?
+
+      @table_exists = connection.table_exists?(TABLE_NAME)
+    rescue StandardError
+      @table_exists = false
+    end
+
+    def state
+      return {} unless table_exists?
+
+      row = connection.select_one(<<~SQL)
+        SELECT last_scale_up_at, last_scale_down_at, updated_at
+        FROM #{TABLE_NAME}
+        WHERE key = #{connection.quote(key)}
+      SQL
+
+      return {} unless row
+
+      {
+        last_scale_up_at: row['last_scale_up_at'],
+        last_scale_down_at: row['last_scale_down_at'],
+        updated_at: row['updated_at']
+      }
+    end
+
+    private
+
+    def connection
+      @config.connection
+    end
+
+    def upsert_state(last_scale_up_at: nil, last_scale_down_at: nil)
+      now = Time.current
+      quoted_key = connection.quote(key)
+      quoted_now = connection.quote(now)
+
+      if last_scale_up_at
+        quoted_time = connection.quote(last_scale_up_at)
+        connection.execute(<<~SQL)
+          INSERT INTO #{TABLE_NAME} (key, last_scale_up_at, created_at, updated_at)
+          VALUES (#{quoted_key}, #{quoted_time}, #{quoted_now}, #{quoted_now})
+          ON CONFLICT (key) DO UPDATE SET
+            last_scale_up_at = EXCLUDED.last_scale_up_at,
+            updated_at = EXCLUDED.updated_at
+        SQL
+      elsif last_scale_down_at
+        quoted_time = connection.quote(last_scale_down_at)
+        connection.execute(<<~SQL)
+          INSERT INTO #{TABLE_NAME} (key, last_scale_down_at, created_at, updated_at)
+          VALUES (#{quoted_key}, #{quoted_time}, #{quoted_now}, #{quoted_now})
+          ON CONFLICT (key) DO UPDATE SET
+            last_scale_down_at = EXCLUDED.last_scale_down_at,
+            updated_at = EXCLUDED.updated_at
+        SQL
+      end
+    end
+  end
+end

data/lib/solid_queue_autoscaler/dashboard/engine.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+require 'action_controller/railtie'
+require 'action_view/railtie'
+
+module SolidQueueAutoscaler
+  module Dashboard
+    # Rails engine that provides the autoscaler dashboard.
+    # Mount at /solid_queue_autoscaler or integrate with Mission Control.
+    #
+    # @example Mount in routes.rb
+    #   mount SolidQueueAutoscaler::Dashboard::Engine => "/solid_queue_autoscaler"
+    #
+    # @example With authentication
+    #   authenticate :user, ->(u) { u.admin? } do
+    #     mount SolidQueueAutoscaler::Dashboard::Engine => "/solid_queue_autoscaler"
+    #   end
+    class Engine < ::Rails::Engine
+      isolate_namespace SolidQueueAutoscaler::Dashboard
+
+      # Engine configuration
+      config.solid_queue_autoscaler_dashboard = ActiveSupport::OrderedOptions.new
+      config.solid_queue_autoscaler_dashboard.title = 'Solid Queue Autoscaler'
+
+      # Configure view paths
+      config.paths['app/views'] = File.expand_path('views', __dir__)
+
+      initializer 'solid_queue_autoscaler.dashboard.view_paths' do
+        ActiveSupport.on_load(:action_controller) do
+          append_view_path File.expand_path('views', __dir__)
+        end
+      end
+
+      initializer 'solid_queue_autoscaler.dashboard.integration' do
+        # Auto-integrate with Mission Control if available
+        ActiveSupport.on_load(:mission_control) do
+          # Register with Mission Control's tab system if available
+        end
+      end
+    end
+
+    # Application controller for dashboard
+    class ApplicationController < ActionController::Base
+      protect_from_forgery with: :exception
+
+      layout 'solid_queue_autoscaler/dashboard/application'
+
+      private
+
+      def autoscaler_status
+        @autoscaler_status ||= SolidQueueAutoscaler::Dashboard.status
+      end
+      helper_method :autoscaler_status
+
+      def events_available?
+        @events_available ||= SolidQueueAutoscaler::Dashboard.events_table_available?
+      end
+      helper_method :events_available?
+    end
+
+    # Main dashboard controller
+    class DashboardController < ApplicationController
+      def index
+        @status = autoscaler_status
+        @stats = SolidQueueAutoscaler::Dashboard.event_stats(since: 24.hours.ago)
+        @recent_events = SolidQueueAutoscaler::Dashboard.recent_events(limit: 10)
+      end
+    end
+
+    # Events controller
+    class EventsController < ApplicationController
+      def index
+        @worker_filter = params[:worker]
+        @events = SolidQueueAutoscaler::Dashboard.recent_events(
+          limit: params.fetch(:limit, 100).to_i,
+          worker_name: @worker_filter
+        )
+        @stats = SolidQueueAutoscaler::Dashboard.event_stats(
+          since: 24.hours.ago,
+          worker_name: @worker_filter
+        )
+      end
+    end
+
+    # Workers controller
+    class WorkersController < ApplicationController
+      def index
+        @workers = autoscaler_status
+      end
+
+      def show
+        worker_name = params[:id].to_sym
+        @worker = SolidQueueAutoscaler::Dashboard.worker_status(worker_name)
+        @events = SolidQueueAutoscaler::Dashboard.recent_events(
+          limit: 20,
+          worker_name: worker_name.to_s
+        )
+      end
+
+      def scale
+        worker_name = params[:id].to_sym
+        @result = SolidQueueAutoscaler.scale!(worker_name)
+        redirect_to worker_path(worker_name), notice: scale_notice(@result)
+      end
+
+      private
+
+      def scale_notice(result)
+        if result.success?
+          if result.scaled?
+            "Scaled from #{result.decision.from} to #{result.decision.to} workers"
+          elsif result.skipped?
+            "Skipped: #{result.skipped_reason}"
+          else
+            "No change needed: #{result.decision&.reason}"
+          end
+        else
+          "Error: #{result.error}"
+        end
+      end
+    end
+  end
+end
+
+# Define routes for the engine
+SolidQueueAutoscaler::Dashboard::Engine.routes.draw do
+  root to: 'dashboard#index'
+
+  resources :events, only: [:index]
+
+  resources :workers, only: %i[index show] do
+    member do
+      post :scale
+    end
+  end
+end