solid_queue_heroku_autoscaler 0.1.0

data/lib/generators/solid_queue_heroku_autoscaler/templates/README

@@ -0,0 +1,41 @@
+===============================================================================
+
+SolidQueueHerokuAutoscaler has been installed!
+
+Next steps:
+
+1. Set environment variables:
+   - HEROKU_API_KEY: Generate with `heroku authorizations:create -d "Solid Queue Autoscaler"`
+   - HEROKU_APP_NAME: Your Heroku app name
+
+2. Run the migration generator for persistent cooldown tracking:
+
+   rails generate solid_queue_heroku_autoscaler:migration
+   rails db:migrate
+
+3. Review config/initializers/solid_queue_autoscaler.rb and adjust thresholds
+
+4. Add the recurring job to config/recurring.yml:
+
+   autoscaler:
+     class: SolidQueueHerokuAutoscaler::AutoscaleJob
+     queue: autoscaler
+     schedule: every 30 seconds
+
+5. Configure a dedicated queue in config/queue.yml:
+
+   queues:
+     - autoscaler
+     - default
+
+   workers:
+     - queues: [autoscaler]
+       threads: 1
+     - queues: [default]
+       threads: 5
+
+6. Test with dry_run mode before enabling in production
+
+For more information, see: https://github.com/reillyse/solid_queue_heroku_autoscaler
+
+===============================================================================

data/lib/generators/solid_queue_heroku_autoscaler/templates/create_solid_queue_autoscaler_state.rb.erb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class CreateSolidQueueAutoscalerState < ActiveRecord::Migration<%= migration_version %>
+  def change
+    create_table :solid_queue_autoscaler_state do |t|
+      t.string :key, null: false
+      t.datetime :last_scale_up_at
+      t.datetime :last_scale_down_at
+
+      t.timestamps
+    end
+
+    add_index :solid_queue_autoscaler_state, :key, unique: true
+  end
+end

data/lib/generators/solid_queue_heroku_autoscaler/templates/initializer.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+SolidQueueHerokuAutoscaler.configure do |config|
+  # Required: Heroku settings
+  # Generate an API key with: heroku authorizations:create -d "Solid Queue Autoscaler"
+  config.heroku_api_key = ENV.fetch('HEROKU_API_KEY', nil)
+  config.heroku_app_name = ENV.fetch('HEROKU_APP_NAME', nil)
+  config.process_type = 'worker'
+
+  # Worker limits
+  config.min_workers = 1
+  config.max_workers = 10
+
+  # Scaling strategy:
+  # :fixed - adds/removes fixed increment/decrement (default)
+  # :proportional - calculates workers based on jobs/latency over threshold
+  config.scaling_strategy = :fixed
+
+  # Scale-up thresholds (trigger scale up when ANY threshold is exceeded)
+  config.scale_up_queue_depth = 100      # Jobs waiting in queue
+  config.scale_up_latency_seconds = 300  # Age of oldest job (5 minutes)
+  config.scale_up_increment = 1          # Workers to add per scale event (fixed strategy)
+
+  # Proportional scaling settings (when scaling_strategy is :proportional)
+  # config.scale_up_jobs_per_worker = 50      # Add 1 worker per 50 jobs over threshold
+  # config.scale_up_latency_per_worker = 60   # Add 1 worker per 60s over threshold
+  # config.scale_down_jobs_per_worker = 50    # Remove 1 worker per 50 jobs under capacity
+
+  # Scale-down thresholds (trigger scale down when ALL thresholds are met)
+  config.scale_down_queue_depth = 10     # Jobs waiting in queue
+  config.scale_down_latency_seconds = 30 # Age of oldest job
+  config.scale_down_decrement = 1        # Workers to remove per scale event
+
+  # Cooldowns (prevent rapid scaling)
+  config.cooldown_seconds = 120 # Default cooldown for both directions
+  # config.scale_up_cooldown_seconds = 60    # Override for scale up
+  # config.scale_down_cooldown_seconds = 180 # Override for scale down
+
+  # Safety settings
+  config.dry_run = Rails.env.development? # Safe default for development
+  config.enabled = Rails.env.production?  # Only enable in production
+
+  # Cooldown persistence (survives dyno restarts)
+  # Requires running: rails generate solid_queue_heroku_autoscaler:migration
+  config.persist_cooldowns = true
+
+  # Optional: Filter to specific queues (nil = all queues)
+  # config.queues = ['default', 'mailers']
+
+  # Optional: Custom logger
+  # config.logger = Rails.logger
+end

data/lib/solid_queue_heroku_autoscaler/adapters/base.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module SolidQueueHerokuAutoscaler
+  module Adapters
+    # Base class for infrastructure platform adapters.
+    #
+    # Subclasses must implement:
+    # - #current_workers - returns Integer count of current workers
+    # - #scale(quantity) - scales to quantity workers, returns new count
+    #
+    # Subclasses may override:
+    # - #name - human-readable adapter name (default: class name)
+    # - #configured? - whether adapter has valid configuration (default: configuration_errors.empty?)
+    # - #configuration_errors - array of configuration error messages (default: [])
+    #
+    # @example Creating a custom adapter
+    #   class MyAdapter < SolidQueueHerokuAutoscaler::Adapters::Base
+    #     def current_workers
+    #       # Return current worker count
+    #     end
+    #
+    #     def scale(quantity)
+    #       return quantity if dry_run?
+    #       # Scale to quantity workers
+    #       quantity
+    #     end
+    #
+    #     def configuration_errors
+    #       errors = []
+    #       errors << 'my_setting is required' if config.my_setting.nil?
+    #       errors
+    #     end
+    #   end
+    class Base
+      # @param config [Configuration] the autoscaler configuration
+      def initialize(config:)
+        @config = config
+      end
+
+      # Returns the current number of workers.
+      #
+      # @return [Integer] current worker count
+      # @raise [NotImplementedError] if not implemented by subclass
+      def current_workers
+        raise NotImplementedError, "#{self.class.name} must implement #current_workers"
+      end
+
+      # Scales to the specified number of workers.
+      #
+      # @param quantity [Integer] desired worker count
+      # @return [Integer] the new worker count
+      # @raise [NotImplementedError] if not implemented by subclass
+      def scale(quantity)
+        raise NotImplementedError, "#{self.class.name} must implement #scale(quantity)"
+      end
+
+      # Human-readable name of the adapter for logging.
+      #
+      # @return [String] adapter name
+      def name
+        self.class.name.split('::').last
+      end
+
+      # Checks if the adapter is properly configured.
+      #
+      # @return [Boolean] true if configured correctly
+      def configured?
+        configuration_errors.empty?
+      end
+
+      # Returns an array of configuration error messages.
+      #
+      # @return [Array<String>] error messages (empty if valid)
+      def configuration_errors
+        []
+      end
+
+      protected
+
+      # @return [Configuration] the autoscaler configuration
+      attr_reader :config
+
+      # @return [Logger] the configured logger
+      def logger
+        config.logger
+      end
+
+      # @return [Boolean] true if dry-run mode is enabled
+      def dry_run?
+        config.dry_run?
+      end
+
+      # Logs a dry-run message at info level.
+      #
+      # @param message [String] the message to log
+      # @return [void]
+      def log_dry_run(message)
+        logger.info("[DRY RUN] #{message}")
+      end
+    end
+  end
+end

data/lib/solid_queue_heroku_autoscaler/adapters/heroku.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require 'platform-api'
+
+module SolidQueueHerokuAutoscaler
+  module Adapters
+    # Heroku adapter using the Heroku Platform API.
+    #
+    # This is the default adapter for the autoscaler.
+    #
+    # Configuration:
+    # - heroku_api_key: Heroku API key (or HEROKU_API_KEY env var)
+    # - heroku_app_name: Heroku app name (or HEROKU_APP_NAME env var)
+    # - process_type: Dyno process type to scale (default: 'worker')
+    #
+    # @example
+    #   SolidQueueHerokuAutoscaler.configure do |config|
+    #     config.heroku_api_key = ENV['HEROKU_API_KEY']
+    #     config.heroku_app_name = ENV['HEROKU_APP_NAME']
+    #     config.process_type = 'worker'
+    #   end
+    class Heroku < Base
+      def current_workers
+        formation = client.formation.info(app_name, process_type)
+        formation['quantity']
+      rescue Excon::Error => e
+        raise HerokuAPIError.new(
+          "Failed to get formation info: #{e.message}",
+          status_code: e.respond_to?(:response) ? e.response&.status : nil,
+          response_body: e.respond_to?(:response) ? e.response&.body : nil
+        )
+      end
+
+      def scale(quantity)
+        if dry_run?
+          log_dry_run("Would scale #{process_type} to #{quantity} dynos")
+          return quantity
+        end
+
+        client.formation.update(app_name, process_type, { quantity: quantity })
+        quantity
+      rescue Excon::Error => e
+        raise HerokuAPIError.new(
+          "Failed to scale #{process_type} to #{quantity}: #{e.message}",
+          status_code: e.respond_to?(:response) ? e.response&.status : nil,
+          response_body: e.respond_to?(:response) ? e.response&.body : nil
+        )
+      end
+
+      def name
+        'Heroku'
+      end
+
+      def configuration_errors
+        errors = []
+        errors << 'heroku_api_key is required' if api_key.nil? || api_key.empty?
+        errors << 'heroku_app_name is required' if app_name.nil? || app_name.empty?
+        errors
+      end
+
+      # Returns the list of all formations for the app.
+      #
+      # @return [Array<Hash>] formation info hashes
+      def formation_list
+        client.formation.list(app_name)
+      rescue Excon::Error => e
+        raise HerokuAPIError.new(
+          "Failed to list formations: #{e.message}",
+          status_code: e.respond_to?(:response) ? e.response&.status : nil,
+          response_body: e.respond_to?(:response) ? e.response&.body : nil
+        )
+      end
+
+      private
+
+      def client
+        @client ||= PlatformAPI.connect_oauth(api_key)
+      end
+
+      def api_key
+        config.heroku_api_key
+      end
+
+      def app_name
+        config.heroku_app_name
+      end
+
+      def process_type
+        config.process_type
+      end
+    end
+  end
+end

data/lib/solid_queue_heroku_autoscaler/adapters/kubernetes.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+module SolidQueueHerokuAutoscaler
+  module Adapters
+    # Kubernetes adapter for scaling Deployment replicas.
+    #
+    # Uses the kubeclient gem to interact with the Kubernetes API.
+    # Supports both in-cluster configuration and kubeconfig file authentication.
+    #
+    # Configuration:
+    # - kubernetes_deployment: Name of the Deployment to scale (or K8S_DEPLOYMENT env var)
+    # - kubernetes_namespace: Namespace of the Deployment (default: 'default', or K8S_NAMESPACE env var)
+    # - kubernetes_context: Kubeconfig context to use (optional, for kubeconfig auth)
+    #
+    # @example In-cluster configuration (running inside a pod)
+    #   SolidQueueHerokuAutoscaler.configure do |config|
+    #     config.adapter_class = SolidQueueHerokuAutoscaler::Adapters::Kubernetes
+    #     config.kubernetes_deployment = 'my-worker'
+    #     config.kubernetes_namespace = 'production'
+    #   end
+    #
+    # @example Using kubeconfig (local development)
+    #   SolidQueueHerokuAutoscaler.configure do |config|
+    #     config.adapter_class = SolidQueueHerokuAutoscaler::Adapters::Kubernetes
+    #     config.kubernetes_deployment = 'my-worker'
+    #     config.kubernetes_namespace = 'default'
+    #     config.kubernetes_context = 'my-cluster-context'
+    #   end
+    class Kubernetes < Base
+      # Kubernetes API path for apps/v1 group
+      APPS_API_VERSION = 'apis/apps/v1'
+
+      def current_workers
+        deployment = apps_client.get_deployment(deployment_name, namespace)
+        deployment.spec.replicas
+      rescue StandardError => e
+        raise KubernetesAPIError.new("Failed to get deployment info: #{e.message}", original_error: e)
+      end
+
+      def scale(quantity)
+        if dry_run?
+          log_dry_run("Would scale deployment #{deployment_name} to #{quantity} replicas in namespace #{namespace}")
+          return quantity
+        end
+
+        patch_body = { spec: { replicas: quantity } }
+        apps_client.patch_deployment(deployment_name, patch_body, namespace)
+        quantity
+      rescue StandardError => e
+        raise KubernetesAPIError.new("Failed to scale deployment #{deployment_name} to #{quantity}: #{e.message}",
+                                     original_error: e)
+      end
+
+      def name
+        'Kubernetes'
+      end
+
+      def configuration_errors
+        errors = []
+        errors << 'kubernetes_deployment is required' if deployment_name.nil? || deployment_name.empty?
+        errors << 'kubernetes_namespace is required' if namespace.nil? || namespace.empty?
+        errors
+      end
+
+      private
+
+      def apps_client
+        @apps_client ||= build_apps_client
+      end
+
+      def build_apps_client
+        require 'kubeclient'
+
+        if in_cluster?
+          build_in_cluster_client
+        else
+          build_kubeconfig_client
+        end
+      end
+
+      def build_in_cluster_client
+        # In-cluster configuration reads from mounted service account
+        auth_options = {
+          bearer_token_file: '/var/run/secrets/kubernetes.io/serviceaccount/token'
+        }
+
+        ssl_options = {
+          ca_file: '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt',
+          verify_ssl: OpenSSL::SSL::VERIFY_PEER
+        }
+
+        api_endpoint = "https://#{kubernetes_host}:#{kubernetes_port}/#{APPS_API_VERSION}"
+
+        Kubeclient::Client.new(
+          api_endpoint,
+          'v1',
+          auth_options: auth_options,
+          ssl_options: ssl_options
+        )
+      end
+
+      def build_kubeconfig_client
+        kubeconfig_path = config.respond_to?(:kubernetes_kubeconfig) ? config.kubernetes_kubeconfig : nil
+        kubeconfig_path ||= ENV['KUBECONFIG'] || File.join(Dir.home, '.kube', 'config')
+
+        kubeconfig = Kubeclient::Config.read(kubeconfig_path)
+        context = kubeconfig.context(kubernetes_context)
+
+        api_endpoint = "#{context.api_endpoint}/#{APPS_API_VERSION}"
+
+        Kubeclient::Client.new(
+          api_endpoint,
+          'v1',
+          ssl_options: context.ssl_options,
+          auth_options: context.auth_options
+        )
+      end
+
+      def in_cluster?
+        # Check if running inside a Kubernetes pod by looking for the service account token
+        File.exist?('/var/run/secrets/kubernetes.io/serviceaccount/token')
+      end
+
+      def kubernetes_host
+        ENV['KUBERNETES_SERVICE_HOST'] || 'kubernetes.default.svc'
+      end
+
+      def kubernetes_port
+        ENV['KUBERNETES_SERVICE_PORT'] || '443'
+      end
+
+      def deployment_name
+        if config.respond_to?(:kubernetes_deployment)
+          config.kubernetes_deployment
+        else
+          ENV.fetch('K8S_DEPLOYMENT', nil)
+        end
+      end
+
+      def namespace
+        ns = if config.respond_to?(:kubernetes_namespace)
+               config.kubernetes_namespace
+             else
+               ENV.fetch('K8S_NAMESPACE', nil)
+             end
+        ns || 'default'
+      end
+
+      def kubernetes_context
+        if config.respond_to?(:kubernetes_context)
+          config.kubernetes_context
+        else
+          ENV.fetch('K8S_CONTEXT', nil)
+        end
+      end
+    end
+  end
+end

data/lib/solid_queue_heroku_autoscaler/adapters.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require_relative 'adapters/base'
+require_relative 'adapters/heroku'
+require_relative 'adapters/kubernetes'
+
+module SolidQueueHerokuAutoscaler
+  # Adapters module provides the plugin architecture for different platforms.
+  #
+  # Built-in adapters:
+  # - Adapters::Heroku (default)
+  # - Adapters::Kubernetes
+  #
+  # See Adapters::Base for creating custom adapters.
+  module Adapters
+    class << self
+      # Internal registry of adapter classes keyed by symbolic name.
+      #
+      # @return [Hash<Symbol, Class>]
+      def registry
+        @registry ||= {
+          heroku: Heroku,
+          kubernetes: Kubernetes,
+          k8s: Kubernetes
+        }
+      end
+
+      # Returns all registered adapter classes.
+      #
+      # @return [Array<Class>]
+      def all
+        registry.values.uniq
+      end
+
+      # Register a new adapter class.
+      #
+      # @param name [Symbol, String] symbolic name (e.g., :aws_ecs)
+      # @param klass [Class] adapter class (subclass of Adapters::Base)
+      # @return [void]
+      def register(name, klass)
+        registry[name.to_sym] = klass
+      end
+
+      # Find an adapter by symbolic name or by class short name.
+      #
+      # @param name [Symbol, String] adapter name (e.g., :heroku, :aws_ecs, 'Heroku')
+      # @return [Class, nil] adapter class or nil if not found
+      def find(name)
+        symbol = name.to_sym
+        return registry[symbol] if registry.key?(symbol)
+
+        name_str = name.to_s.downcase
+        all.find { |adapter| adapter.name.split('::').last.downcase == name_str }
+      end
+    end
+  end
+end

data/lib/solid_queue_heroku_autoscaler/advisory_lock.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require 'zlib'
+
+module SolidQueueHerokuAutoscaler
+  class AdvisoryLock
+    attr_reader :lock_key, :timeout
+
+    def initialize(lock_key: nil, timeout: nil, config: nil)
+      @config = config || SolidQueueHerokuAutoscaler.config
+      @lock_key = lock_key || @config.lock_key
+      @timeout = timeout || @config.lock_timeout_seconds
+      @lock_acquired = false
+    end
+
+    def with_lock
+      acquire!
+      yield
+    ensure
+      release
+    end
+
+    def try_lock
+      return false if @lock_acquired
+
+      result = connection.select_value(
+        "SELECT pg_try_advisory_lock(#{lock_id})"
+      )
+      @lock_acquired = [true, 't'].include?(result)
+      @lock_acquired
+    end
+
+    def acquire!
+      return true if @lock_acquired
+
+      result = connection.select_value(
+        "SELECT pg_try_advisory_lock(#{lock_id})"
+      )
+      @lock_acquired = [true, 't'].include?(result)
+
+      raise LockError, "Could not acquire advisory lock '#{lock_key}' (id: #{lock_id})" unless @lock_acquired
+
+      true
+    end
+
+    def release
+      return false unless @lock_acquired
+
+      connection.execute("SELECT pg_advisory_unlock(#{lock_id})")
+      @lock_acquired = false
+      true
+    end
+
+    def locked?
+      @lock_acquired
+    end
+
+    private
+
+    def connection
+      @config.connection
+    end
+
+    def lock_id
+      @lock_id ||= begin
+        hash = Zlib.crc32(lock_key.to_s)
+        hash & 0x7FFFFFFF
+      end
+    end
+  end
+end

data/lib/solid_queue_heroku_autoscaler/autoscale_job.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module SolidQueueHerokuAutoscaler
+  class AutoscaleJob < ActiveJob::Base
+    queue_as :autoscaler
+    discard_on ConfigurationError
+
+    # Scale a specific worker type, or all workers if :all is passed
+    # @param worker_name [Symbol] The worker type to scale (:default, :critical_worker, etc.)
+    #                             Pass :all to scale all registered workers
+    def perform(worker_name = :default)
+      if worker_name == :all
+        perform_scale_all
+      else
+        perform_scale_one(worker_name)
+      end
+    end
+
+    private
+
+    def perform_scale_one(worker_name)
+      result = SolidQueueHerokuAutoscaler.scale!(worker_name)
+
+      if result.success?
+        log_success(result, worker_name)
+      else
+        log_failure(result, worker_name)
+        raise result.error if result.error
+      end
+
+      result
+    end
+
+    def perform_scale_all
+      results = SolidQueueHerokuAutoscaler.scale_all!
+
+      results.each do |worker_name, result|
+        if result.success?
+          log_success(result, worker_name)
+        else
+          log_failure(result, worker_name)
+        end
+      end
+
+      # Raise the first error encountered, if any
+      failed_result = results.values.find { |r| !r.success? && r.error }
+      raise failed_result.error if failed_result
+
+      results
+    end
+
+    def log_success(result, worker_name)
+      worker_label = worker_name == :default ? '' : "[#{worker_name}] "
+      if result.scaled?
+        Rails.logger.info(
+          "[AutoscaleJob] #{worker_label}Scaled workers: #{result.decision.from} -> #{result.decision.to} " \
+          "(#{result.decision.reason})"
+        )
+      elsif result.skipped?
+        Rails.logger.debug("[AutoscaleJob] #{worker_label}Skipped: #{result.skipped_reason}")
+      else
+        Rails.logger.debug("[AutoscaleJob] #{worker_label}No change: #{result.decision&.reason}")
+      end
+    end
+
+    def log_failure(result, worker_name)
+      worker_label = worker_name == :default ? '' : "[#{worker_name}] "
+      Rails.logger.error("[AutoscaleJob] #{worker_label}Failed: #{result.error&.message}")
+    end
+  end
+end