sidekiq_autoscale 0.2.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: db767100bab6dcb54dcea794f7a5fb3b4e1ba925330ece323350fd6256399899
+  data.tar.gz: 4bb7d27d12ec1953e3cfdc1b37cccabbde656e444abe8e6cf4ecafad838e9f94
+SHA512:
+  metadata.gz: 4bd8d6c970aac521f6cfc52bcb4fc8834058ab2bb23e68bf903eb51e8480e0c3b40b69bda22955e1466985d77a57067b94916ba61b10220de1d3836a0c3dd70f
+  data.tar.gz: a4818632e8f52a4955beb4014f113f3b5f6f64f38c9f6503229a60b97fae38e77361b99fb1297b8590e62fcacc395f9f9fe570357a46f600ef31033bd72aca6e

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2019 Traction Guest
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,152 @@
+[![Maintainability](https://api.codeclimate.com/v1/badges/505a50e09d651c0423fd/maintainability)](https://codeclimate.com/github/tractionguest/sidekiq_autoscaling/maintainability)
+
+[![Test Coverage](https://api.codeclimate.com/v1/badges/505a50e09d651c0423fd/test_coverage)](https://codeclimate.com/github/tractionguest/sidekiq_autoscaling/test_coverage)
+
+# SidekiqAutoscale
+
+A simple gem to manage autoscaling of Sidekiq worker pools
+
+## Usage
+
+
+
+## Installation
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'sidekiq_autoscale'
+```
+
+And then execute:
+```bash
+$ bundle install
+```
+
+Install the initializer:
+```bash
+$ rails g sidekiq_autoscale:install
+```
+
+This will create a `config/initializers/sidekiq_autoscale.rb` file.
+
+### Installing middleware in Sidekiq chain
+
+Add the scaling middleware to your Sidekiq configuration
+
+```ruby
+Sidekiq.configure_server do |config|
+  .....
+  config.server_middleware do |chain|
+    chain.add SidekiqScaling::Middleware
+  end
+end
+
+```
+
+
+## Configuration
+
+All configuration can be done in a `SidekiqAutoscale.configure` block, either in a standalone initializer or in `<environment>.rb` files.
+
+Standard configuration looks like this:
+
+```ruby
+SidekiqAutoscale.configure do |config|
+  config.cache = Rails.cache
+  config.logger = Rails.logger
+  config.redis = # Put a real Redis client instance here
+
+  # Number of workers will never go below this threshold
+  config.min_workers = 1
+
+  # Number of workers will never go above this threshold
+  config.max_workers = 20
+
+  # The up and down thresholds used by all scaling strategies
+  config.scale_down_threshold = 1.0
+  config.scale_up_threshold = 5.0
+
+  # Current strategies are:
+  #  :oldest_job - scales based on the age (in seconds) of the oldest job in any Sidekiq queue
+  #  :delay_based - scales based on the average age (in seconds) all jobs run in the last minute
+  #  :linear - scales based the total number of jobs in all queues, divided by the number of workers
+  #  :base - do not scale, ever
+  config.strategy = :base
+
+  # Current adapters are:
+  #  :nil - scaling events do nothing
+  #  :heroku - scale a Heroku dyno
+
+  config.adapter = :heroku
+
+  # Any configuration required for the selected adapter
+  # Heroku requires the following:
+  config.adapter_config = {
+                            api_key: "HEROKU_API_KEY", 
+                            worker_dyno_name: "DYNO_WORKER_NAME", 
+                            app_name: "HEROKU_APP_NAME"
+                          }
+  # Kubernetes requires the following:
+  config.adapter_config = {
+                            deployment_name: "myapp-sidekiq", 
+                          }
+
+  # The minimum amount of time to wait between scaling events
+  # Useful to tweak based on how long it takes for a new worker
+  # to spin up and start working on the pool
+  # config.min_scaling_interval = 5.minutes.to_i
+  
+  # The number of workers to change in a scaling event
+  # config.scale_by = 1
+
+  # This proc will be called on a scaling event
+  # config.on_scaling_event = Proc.new { |event| Rails.logger.info event.to_json }
+  
+  # This proc will be called when a scaling event errors out
+  # By default, nothing happens
+  # config.on_scaling_error = Proc.new { |error| Rails.logger.error error.to_json }
+end
+```
+
+## Kubernetes
+
+This gem can scale a Kubernetes `Deployment` object that Sidekiq is running in.
+
+Doing so requires the following RBAC config:
+
+```
+
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: myapp-sidekiq
+rules:
+  - apiGroups: ["apps"]
+    resources: ["deployments"]
+    verbs: ["get", "patch"]
+
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: sidekiq
+subjects:
+  - kind: ServiceAccount
+    name: myapp-sidekiq
+roleRef:
+  kind: Role
+  name: myapp-sidekiq
+  apiGroup: rbac.authorization.k8s.io
+
+---
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  name: myapp-sidekiq
+```
+
+Then assign your sidekiq deployment the `myapp-sidekiq` service account.
+
+## License
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+begin
+  require "bundler/setup"
+rescue LoadError
+  puts "You must `gem install bundler` and `bundle install` to run rake tasks"
+end
+
+require "rdoc/task"
+
+RDoc::Task.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = "rdoc"
+  rdoc.title    = "SidekiqAutoscale"
+  rdoc.options << "--line-numbers"
+  rdoc.rdoc_files.include("README.md")
+  rdoc.rdoc_files.include("lib/**/*.rb")
+end
+
+require "bundler/gem_tasks"
+
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.pattern = "test/**/*_test.rb"
+  t.verbose = false
+end
+
+task default: :test

data/lib/generators/sidekiq_autoscale/install/install_generator.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+
+module SidekiqAutoscale
+  class InstallGenerator < Rails::Generators::Base
+    source_root File.expand_path("templates", __dir__)
+
+    def generate_install
+      template "sidekiq_autoscale_initializer_template.template", "config/initializers/sidekiq_autoscale.rb"
+    end
+  end
+end

data/lib/generators/sidekiq_autoscale/install/templates/sidekiq_autoscale_initializer_template.template

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+SidekiqAutoscale.configure do |config|
+  config.cache = Rails.cache
+  config.logger = Rails.logger
+  config.redis_client = # Put a real Redis client instance here
+
+  # Number of workers will never go below this threshold
+  config.min_workers = 1
+
+  # Number of workers will never go above this threshold
+  config.max_workers = 20
+
+  # The up and down thresholds used by all scaling strategies
+  config.scale_down_threshold = 1.0
+  config.scale_up_threshold = 5.0
+
+  # Current strategies are:
+  #  :oldest_job - scales based on the age (in seconds) of the oldest job in any Sidekiq queue
+  #  :delay_based - scales based on the average age (in seconds) all jobs run in the last minute
+  #  :linear - scales based the total number of jobs in all queues, divided by the number of workers
+  #  :base - do not scale, ever
+  config.strategy = :base
+
+  # Current adapters are:
+  #  :nil - scaling events do nothing
+  #  :heroku - scale a Heroku dyno
+
+  config.adapter = :heroku
+
+  # Any configuration required for the selected adapter
+  # Heroku requires the following:
+  config.adapter_config = {
+                            api_key: "HEROKU_API_KEY", 
+                            worker_dyno_name: "DYNO_WORKER_NAME", 
+                            app_name: "HEROKU_APP_NAME"
+                          }
+
+  # The minimum amount of time to wait between scaling events
+  # Useful to tweak based on how long it takes for a new worker
+  # to spin up and start working on the pool
+  # config.min_scaling_interval = 5.minutes.to_i
+  
+  # The number of workers to change in a scaling event
+  # config.scale_by = 1
+
+  # This proc will be called on a scaling event
+  # config.on_scaling_event = Proc.new { |event| Rails.logger.info event.to_json }
+  
+  # This proc will be called when a scaling event errors out
+  # By default, nothing happens
+  # config.on_scaling_error = Proc.new { |error| Rails.logger.error error.to_json }
+end

data/lib/sidekiq_autoscale/adapters/heroku_adapter.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module SidekiqAutoscale
+  class HerokuAdapter
+    def initialize
+      require "platform-api"
+      @app_name = SidekiqAutoscale.adapter_config[:app_name]
+      @dyno_name = SidekiqAutoscale.adapter_config[:worker_dyno_name]
+      @client = PlatformAPI.connect_oauth(SidekiqAutoscale.adapter_config[:api_key])
+    end
+
+    def worker_count
+      @client.formation.list(@app_name)
+             .select {|i| i["type"] == @dyno_name }
+             .map {|i| i["quantity"] }
+             .reduce(0, &:+)
+    rescue Excon::Errors::Error => e
+      SidekiqAutoscale.on_scaling_error(e)
+      0
+    end
+
+    def worker_count=(val)
+      return if val == worker_count
+
+      SidekiqAutoscale.logger.info("[SIDEKIQ_SCALE][HEROKU_ACTION] Setting new worker count to #{val} (is currenly #{worker_count})")
+      @client.formation.update(@app_name, @dyno_name, quantity: val)
+    rescue Excon::Errors::Error, Heroku::API::Errors::Error => e
+      SidekiqAutoscale.on_scaling_error(e)
+    end
+  end
+end

data/lib/sidekiq_autoscale/adapters/kubernetes_adapter.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module SidekiqAutoscale
+  # Scale a Kubernetes deployment object.
+  class KubernetesAdapter
+    def initialize
+      require "k8s-ruby"
+
+      namespace = File.read("/run/secrets/kubernetes.io/serviceaccount/namespace")
+      client = K8s::Client.autoconfig
+
+      @deployment_name = SidekiqAutoscale.adapter_config[:deployment_name]
+      @resources = client.api("apps/v1").resource("deployments", namespace: namespace)
+    end
+
+    def worker_count
+      @resources.get(@deployment_name).spec.replicas
+    rescue Excon::Errors::Error, K8s::Error, K8s::Error::Forbidden => e
+      SidekiqAutoscale.on_scaling_error(e)
+      0
+    end
+
+    def worker_count=(val)
+      return if val == worker_count
+
+      SidekiqAutoscale.logger.info("[SIDEKIQ_SCALE][KUBERNETES_ACTION] Setting new worker count to #{val} (is currenly #{worker_count})")
+      @resources.merge_patch(@deployment_name, spec: {replicas: val})
+    rescue Excon::Errors::Error, K8s::Error, K8s::Error::Forbidden => e
+      SidekiqAutoscale.on_scaling_error(e)
+    end
+  end
+end

data/lib/sidekiq_autoscale/adapters/nil_adapter.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module SidekiqAutoscale
+  class NilAdapter
+    def initialize
+      @sidekiq_adapter = ::SidekiqAutoscale::SidekiqInterface.new
+    end
+
+    def worker_count
+      @sidekiq_adapter.total_workers
+    end
+
+    def worker_count=(val)
+      SidekiqAutoscale.logger.debug("Attempting to autoscale sidekiq to #{val} workers")
+    end
+  end
+end

data/lib/sidekiq_autoscale/config/shared_configs.rb

@@ -0,0 +1,201 @@
+# frozen_string_literal: true
+
+module SidekiqAutoscale
+  module Config
+    module SharedConfigs
+      LOG_TAG = "[SIDEKIQ_SCALING]"
+
+      attr_writer :config
+
+      def config
+        @config ||= ActiveSupport::OrderedOptions.new
+      end
+
+      def strategy
+        config.strategy || :base
+      end
+
+      def strategy_klass
+        @strategy_klass ||= begin
+          known_strats = [
+            ::SidekiqAutoscale::Strategies::BaseScaling,
+            ::SidekiqAutoscale::Strategies::DelayScaling,
+            ::SidekiqAutoscale::Strategies::OldestJobScaling,
+            ::SidekiqAutoscale::Strategies::LinearScaling,
+            ::SidekiqAutoscale::Strategies::DynamicLatencyScaling
+
+          ]
+          strat_klass_name = known_strats.map(&:to_s).find {|i| i.end_with?("#{strategy.to_s.camelize}Scaling") }
+          if strat_klass_name.nil?
+            raise ::SidekiqAutoscale::Exception.new <<~LOG
+              #{LOG_TAG} Unknown scaling strategy: [#{strategy.to_s.camelize}Scaling]")
+            LOG
+          end
+
+          strat_klass_name.constantize.new
+        end
+      end
+
+      def adapter
+        config.adapter || :nil
+      end
+
+      def adapter_klass
+        @adapter_klass ||= begin
+          known_adapters = [::SidekiqAutoscale::NilAdapter,
+                            ::SidekiqAutoscale::HerokuAdapter,
+                            ::SidekiqAutoscale::KubernetesAdapter].freeze
+          adapter_klass_name = known_adapters.map(&:to_s).find {|i| i.end_with?("#{adapter.to_s.camelize}Adapter") }
+          if adapter_klass_name.nil?
+            raise ::SidekiqAutoscale::Exception.new("#{LOG_TAG} Unknown scaling adapter: [#{adapter.to_s.camelize}Adapter]")
+          end
+
+          adapter_klass_name.constantize.new
+        end
+      end
+
+      def adapter_config
+        config.adapter_config
+      end
+
+      def scale_up_threshold
+        (config.scale_up_threshold ||= begin
+          validate_scaling_thresholds
+          validated_scale_up_threshold
+        end).to_f
+      end
+
+      def scale_down_threshold
+        (config.scale_down_threshold ||= begin
+          validate_scaling_thresholds
+          validated_scale_down_threshold
+        end).to_f
+      end
+
+      def max_workers
+        (@max_workers ||= begin
+          validate_worker_set
+          validated_max_workers
+        end).to_i
+      end
+
+      def min_workers
+        (@min_workers ||= begin
+          validate_worker_set
+          validated_min_workers
+        end).to_i
+      end
+
+      def scale_by
+        (config.scale_by || ENV.fetch("SIDEKIQ_AUTOSCALE_SCALE_BY", 1)).to_i
+      end
+
+      def min_scaling_interval
+        (config.min_scaling_interval || 5.minutes).to_i
+      end
+
+      def redis_client
+        raise ::SidekiqAutoscale::Exception.new("No Redis client defined") unless config.redis_client
+
+        config.redis_client
+      end
+
+      def logger
+        config.logger ||= Rails.logger
+      end
+
+      def cache
+        config.cache ||= ActiveSupport::Cache::NullStore.new
+      end
+
+      def on_scaling_error(e)
+        logger.error(e)
+        return unless config.on_scaling_error.respond_to?(:call)
+
+        config.on_scaling_error.call(e)
+      end
+
+      def on_scaling_event(event)
+        details = config.to_h.slice(:strategy,
+                                    :adapter,
+                                    :scale_up_threshold,
+                                    :scale_down_threshold,
+                                    :max_workers,
+                                    :min_workers,
+                                    :scale_by,
+                                    :min_scaling_interval)
+
+        on_head_bump(details.merge(event)) if event[:target_workers] == max_workers
+        on_toe_stub(details.merge(event)) if event[:target_workers] == min_workers
+
+        return unless config.on_scaling_event.respond_to?(:call)
+
+        config.on_scaling_event.call(details.merge(event))
+      end
+
+      def on_head_bump(event)
+        return unless config.on_head_bump.respond_to?(:call)
+
+        config.on_head_bump.call(event)
+      end
+
+      def on_toe_stub(event)
+        return unless config.on_toe_stub.respond_to?(:call)
+
+        config.on_toe_stub.call(event)
+      end
+
+      def sidekiq_interface
+        @sidekiq_interface ||= ::SidekiqAutoscale::SidekiqInterface.new
+      end
+
+      def lock_manager
+        config.lock_manager ||= ::Redlock::Client.new(Array.wrap(redis_client),
+                                                      retry_count:   3,
+                                                      retry_delay:   200,
+                                                      retry_jitter:  50,
+                                                      redis_timeout: 0.1)
+      end
+
+      def lock_time
+        config.lock_time || 5_000
+      end
+
+      private
+
+      def validate_worker_set
+        ex_klass = ::SidekiqAutoscale::Exception
+        raise ex_klass.new("No max workers set") unless validated_max_workers.positive?
+        raise ex_klass.new("No min workers set") unless validated_min_workers.positive?
+        if validated_max_workers < validated_min_workers
+          raise ex_klass.new("Max workers must be higher than min workers")
+        end
+      end
+
+      def validate_scaling_thresholds
+        ex_klass = ::SidekiqAutoscale::Exception
+        raise ex_klass.new("No scale up threshold set") unless validated_scale_up_threshold.positive?
+        raise ex_klass.new("No scale down threshold set") unless validated_scale_down_threshold.positive?
+        if validated_scale_up_threshold < validated_scale_down_threshold
+          raise ex_klass.new("Scale up threshold must be higher than scale down threshold")
+        end
+      end
+
+      def validated_scale_up_threshold
+        (config.scale_up_threshold || ENV.fetch("SIDEKIQ_AUTOSCALE_UP_THRESHOLD", 5.0)).to_f
+      end
+
+      def validated_scale_down_threshold
+        (config.scale_down_threshold || ENV.fetch("SIDEKIQ_AUTOSCALE_DOWN_THRESHOLD", 1.0)).to_f
+      end
+
+      def validated_max_workers
+        (config.max_workers || ENV.fetch("SIDEKIQ_AUTOSCALE_MAX_WORKERS", 10)).to_i
+      end
+
+      def validated_min_workers
+        (config.min_workers || ENV.fetch("SIDEKIQ_AUTOSCALE_MIN_WORKERS", 1)).to_i
+      end
+    end
+  end
+end

data/lib/sidekiq_autoscale/exception.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module SidekiqAutoscale
+  class Exception < ::StandardError
+  end
+end

data/lib/sidekiq_autoscale/middleware.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module SidekiqAutoscale
+  class Middleware
+    LAST_SCALED_AT_EVENT_KEY = "sidekiq_autoscaling:last_scaled_at"
+    SCALING_LOCK_KEY = "sidekiq_autoscaling:scaling_lock"
+    LOG_TAG = "[SIDEKIQ_SCALE][SCALING_EVENT]"
+    WORKER_COUNT_KEY = "sidekiq_autoscaling/current_worker_count"
+
+    # @param [Object] worker the worker instance
+    # @param [Hash] job the full job payload
+    #   * @see https://github.com/mperham/sidekiq/wiki/Job-Format
+    # @param [String] queue the name of the queue the job was pulled from
+    # @yield the next middleware in the chain or worker `perform` method
+    # @return [Void]
+    def call(_worker_class, job, _queue)
+      # In case the scaling strategy needs to record job-specific stuff before it runs
+      SidekiqAutoscale.strategy_klass.log_job(job)
+      yield # Run the job, THEN scale the cluster
+      begin
+        return unless SidekiqAutoscale.strategy_klass.workload_change_needed?(job)
+
+        direction = SidekiqAutoscale.strategy_klass.scaling_direction(job)
+        new_worker_count = worker_count + (SidekiqAutoscale.scale_by * direction)
+
+        set_worker_count(new_worker_count, event_id: job["jid"], direction: direction)
+      rescue StandardError => e
+        SidekiqAutoscale.logger.error(e)
+        SidekiqAutoscale.on_scaling_error(e)
+      end
+    end
+
+    private
+
+    def worker_count
+      SidekiqAutoscale.cache.fetch(WORKER_COUNT_KEY, expires_in: 1.minute) do
+        SidekiqAutoscale.adapter_klass.worker_count
+      end
+    end
+
+    def set_worker_count(n, event_id: SecureRandom.hex, direction:)
+      clamped = n.clamp(SidekiqAutoscale.min_workers, SidekiqAutoscale.max_workers)
+
+      SidekiqAutoscale.lock_manager.lock(SCALING_LOCK_KEY, SidekiqAutoscale.lock_time) do |locked|
+        # Not awesome, but gotta handle the initial nil case
+        last_scaled_at = SidekiqAutoscale.redis_client.get(LAST_SCALED_AT_EVENT_KEY).to_f
+        SidekiqAutoscale.logger.debug <<~LOG
+          #{LOG_TAG}[#{event_id}] Concurrency lock obtained: #{locked}"
+          Last scaled [#{Time.current.to_i - last_scaled_at.to_i}] seconds ago"
+          Scaling every [#{SidekiqAutoscale.min_scaling_interval}] seconds"
+        LOG
+
+        if locked && (last_scaled_at < SidekiqAutoscale.min_scaling_interval.seconds.ago.to_f)
+          SidekiqAutoscale.adapter_klass.worker_count = clamped
+          SidekiqAutoscale.cache.delete(WORKER_COUNT_KEY)
+          SidekiqAutoscale.redis_client.set(LAST_SCALED_AT_EVENT_KEY, Time.current.to_f)
+          SidekiqAutoscale.on_scaling_event(
+            direction:            direction,
+            target_workers:       clamped,
+            event_id:             event_id,
+            current_worker_count: worker_count,
+            last_scaled_at:       last_scaled_at
+          )
+        else
+          SidekiqAutoscale.logger.debug("#{LOG_TAG}[#{event_id}] ***NOT SCALING***")
+        end
+
+        SidekiqAutoscale.logger.debug("#{LOG_TAG}[#{event_id}] RELEASING LOCK #{locked}") if locked
+      end
+    end
+  end
+end

data/lib/sidekiq_autoscale/railtie.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require "rails"
+
+module SidekiqAutoscale
+  class Railtie < ::Rails::Railtie
+    config.sidekiq_autoscale = ActiveSupport::OrderedOptions.new
+
+    config.after_initialize do
+      SidekiqAutoscale.logger.info <<~LOG
+        [SIDEKIQ_SCALE] Scaling strategy: #{SidekiqAutoscale.strategy}
+        [SIDEKIQ_SCALE] Min workers: #{SidekiqAutoscale.min_workers}
+        [SIDEKIQ_SCALE] Max workers: #{SidekiqAutoscale.max_workers}
+        [SIDEKIQ_SCALE] Scaling by: #{SidekiqAutoscale.scale_by}
+        [SIDEKIQ_SCALE] Provider adapter: #{SidekiqAutoscale.adapter}
+      LOG
+    end
+  end
+end

data/lib/sidekiq_autoscale/sidekiq_interface.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require "sidekiq/api"
+
+module SidekiqAutoscale
+  class SidekiqInterface
+    def total_queue_size
+      queue_names.map {|q| ::Sidekiq::Queue.new(q).size }.reduce(0, &:+)
+    end
+
+    def queue_names
+      ::Sidekiq::Queue.all.map(&:name)
+    end
+
+    def busy_threads
+      ::Sidekiq::Workers.new.map {|_, thread, _| thread }.uniq.size
+    end
+
+    def latency
+      queue_names.map {|q| ::Sidekiq::Queue.new(q).latency }.max
+    end
+
+    def total_workers
+      process_set.size
+    end
+
+    def total_threads
+      process_set.map {|w| w["concurrency"] }.reduce(0, &:+)
+    end
+
+    def available_threads
+      total_threads - busy_threads
+    end
+
+    def youngest_worker
+      process_set.map {|w| w["started_at"] }.max
+    end
+
+    private
+
+    def process_set
+      @process_set ||= ::Sidekiq::ProcessSet.new
+    end
+  end
+end

data/lib/sidekiq_autoscale/strategies/base_scaling.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module SidekiqAutoscale
+  module Strategies
+    class BaseScaling
+      # This strategy doesn't care about individual job metrics
+      def log_job(_job); end
+
+      def workload_change_needed?(_job)
+        false
+      end
+
+      def scaling_direction(_job)
+        0
+      end
+
+      private
+
+      def scale_up_threshold
+        SidekiqAutoscale.scale_up_threshold
+      end
+
+      def scale_down_threshold
+        SidekiqAutoscale.scale_down_threshold
+      end
+    end
+  end
+end

data/lib/sidekiq_autoscale/strategies/delay_scaling.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module SidekiqAutoscale
+  module Strategies
+    class DelayScaling < BaseScaling
+      SAMPLE_RANGE = 60
+      DELAY_LOG_KEY = "sidekiq_autoscaling:delay_log"
+      DELAY_AVERAGE_CACHE_KEY = "sidekiq_autoscaling:delay_average"
+      LOG_TAG = "[SIDEKIQ_SCALE][DELAY_SCALING]"
+
+      def log_job(job)
+        timestamp = Time.current.to_f
+
+        # Gotta do it this way so that each entry is guaranteed to be unique
+        zset_payload = {delay: (timestamp - job["enqueued_at"]), jid: job["jid"]}.to_json
+
+        # Redis zadd runs in O(log(N)) time, so this should be threaded to avoid blocking
+        # Also, it should be connection-pooled, but I can't remember if we're using
+        # redis connection pooling anywhere
+        Thread.new {
+          SidekiqAutoscale.redis_client.zadd(DELAY_LOG_KEY, timestamp, zset_payload)
+        }
+      end
+
+      def workload_change_needed?(_job)
+        workload_too_high? || workload_too_low?
+      end
+
+      def scaling_direction(_job)
+        return -1 if workload_too_low?
+        return 1 if workload_too_high?
+
+        0
+      end
+
+      private
+
+      def delay_average
+        # Only calculate this once every minute - this operation isn't very efficient
+        # We may want to offload it to another Redis DB number, which will be just delay keys
+        SidekiqAutoscale.cache.fetch(DELAY_AVERAGE_CACHE_KEY, expires_in: SAMPLE_RANGE) do
+          # Delete old scores that won't be included in the metric
+          SidekiqAutoscale.redis_client.zremrangebyscore(DELAY_LOG_KEY, 0, SAMPLE_RANGE.ago.to_f)
+          vals = SidekiqAutoscale.redis_client.zrange(DELAY_LOG_KEY, 0, -1).map {|i| JSON.parse(i)["delay"].to_f }
+          return 0 if vals.empty?
+
+          vals.instance_eval { reduce(:+) / size.to_f }
+        rescue JSON::ParserError => e
+          SidekiqAutoscale.logger.error(e)
+          SidekiqAutoscale.logger.error(e.backtrace.join("\n"))
+          return 0
+        end
+      end
+
+      def workload_too_high?
+        too_high = delay_average > SidekiqAutoscale.scale_up_threshold
+        SidekiqAutoscale.logger.debug("#{LOG_TAG} Workload too high") if too_high
+        SidekiqAutoscale.logger.debug("#{LOG_TAG} Current average delay: #{delay_average}, max allowed: #{SidekiqAutoscale.scale_up_threshold}")
+        too_high
+      end
+
+      def workload_too_low?
+        too_low = delay_average < SidekiqAutoscale.scale_down_threshold
+        SidekiqAutoscale.logger.debug("#{LOG_TAG} Workload too low") if too_low
+        SidekiqAutoscale.logger.debug("#{LOG_TAG} Current average delay: #{delay_average}, min allowed: #{SidekiqAutoscale.scale_down_threshold}")
+        too_low
+      end
+
+      def delay_array; end
+    end
+  end
+end

data/lib/sidekiq_autoscale/strategies/dynamic_latency_scaling.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module SidekiqAutoscale
+  module Strategies
+    class DynamicLatencyScaling < BaseScaling
+      LOG_TAG = "[SIDEKIQ_SCALE][DYNAMIC_LATENCY_SCALING]"
+      def workload_change_needed?(_job)
+        workload_too_high? || workload_too_low?
+      end
+
+      def scaling_direction(_job)
+        return -1 if workload_too_low?
+        return [scale_up_factor.to_i, 1].max if workload_too_high?
+
+        0
+      end
+
+      private
+
+      def scale_up_factor
+        1 + (latency - scale_up_threshold) / dynamic_multiple_base
+      end
+
+      def dynamic_multiple_base
+        @dynamic_multiple_base ||= scale_up_threshold - scale_down_threshold
+      end
+
+      def workload_too_high?
+        too_high = latency > scale_up_threshold
+        SidekiqAutoscale.logger.debug("#{LOG_TAG} Workload too high") if too_high
+        SidekiqAutoscale.logger.debug("#{LOG_TAG} Current average delay: #{latency}, max allowed: #{scale_up_threshold}")
+        too_high
+      end
+
+      def workload_too_low?
+        too_low = latency < scale_down_threshold
+        SidekiqAutoscale.logger.debug("#{LOG_TAG} Workload too low") if too_low
+        SidekiqAutoscale.logger.debug("#{LOG_TAG} Current average delay: #{latency}, min allowed: #{scale_down_threshold}")
+        too_low
+      end
+
+      def latency
+        SidekiqAutoscale.sidekiq_interface.latency
+      end
+    end
+  end
+end

data/lib/sidekiq_autoscale/strategies/linear_scaling.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module SidekiqAutoscale
+  module Strategies
+    class LinearScaling < BaseScaling
+      LOG_TAG = "[SIDEKIQ_SCALE][LINEAR_SCALING]"
+
+      def workload_change_needed?(_job)
+        workload_too_high? || workload_too_low?
+      end
+
+      def scaling_direction(_job)
+        return 1 if workload_too_high?
+        return -1 if workload_too_low?
+
+        0
+      end
+
+      private
+
+      # Remove available threads from total queue size in case there's pending
+      # tasks that are still spinning up,
+      def scheduled_jobs_per_thread
+        ((SidekiqAutoscale.sidekiq_interface.total_queue_size - SidekiqAutoscale.sidekiq_interface.available_threads).to_f / SidekiqAutoscale.sidekiq_interface.total_threads.to_f)
+      end
+
+      def workload_too_high?
+        too_high = scheduled_jobs_per_thread > SidekiqAutoscale.scale_up_threshold
+        if too_high
+          SidekiqAutoscale.logger.debug("#{LOG_TAG} Workload too low [Scheduled: #{scheduled_jobs_per_thread}, Max: #{SidekiqAutoscale.scale_up_threshold}]")
+        end
+        too_high
+      end
+
+      def workload_too_low?
+        too_low = scheduled_jobs_per_thread < SidekiqAutoscale.scale_down_threshold
+        if too_low
+          SidekiqAutoscale.logger.debug("#{LOG_TAG} Workload too low [Scheduled: #{scheduled_jobs_per_thread}, Min: #{SidekiqAutoscale.scale_down_threshold}]")
+        end
+        too_low
+      end
+    end
+  end
+end

data/lib/sidekiq_autoscale/strategies/oldest_job_scaling.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module SidekiqAutoscale
+  module Strategies
+    class OldestJobScaling < BaseScaling
+      LOG_TAG = "[SIDEKIQ_SCALE][OLDEST_JOB_SCALING]"
+      def workload_change_needed?(_job)
+        workload_too_high? || workload_too_low?
+      end
+
+      def scaling_direction(_job)
+        return -1 if workload_too_low?
+        return 1 if workload_too_high?
+
+        0
+      end
+
+      private
+
+      def workload_too_high?
+        too_high = SidekiqAutoscale.sidekiq_interface.latency > SidekiqAutoscale.scale_up_threshold
+
+        SidekiqAutoscale.logger.debug("#{LOG_TAG} Workload too high") if too_high
+        SidekiqAutoscale.logger.debug("#{LOG_TAG} Current average delay: #{SidekiqAutoscale.sidekiq_interface.latency}, max allowed: #{SidekiqAutoscale.scale_up_threshold}")
+        too_high
+      end
+
+      def workload_too_low?
+        too_low = SidekiqAutoscale.sidekiq_interface.latency < SidekiqAutoscale.scale_down_threshold
+        SidekiqAutoscale.logger.debug("#{LOG_TAG} Workload too low") if too_low
+        SidekiqAutoscale.logger.debug("#{LOG_TAG} Current average delay: #{SidekiqAutoscale.sidekiq_interface.latency}, min allowed: #{SidekiqAutoscale.scale_down_threshold}")
+        too_low
+      end
+    end
+  end
+end

data/lib/sidekiq_autoscale/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module SidekiqAutoscale
+  VERSION = "0.2.2"
+end

data/lib/sidekiq_autoscale.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require "redlock"
+require "sidekiq/api"
+require "active_support/all"
+
+require "sidekiq_autoscale/railtie"
+require "sidekiq_autoscale/exception"
+require "sidekiq_autoscale/sidekiq_interface"
+require "sidekiq_autoscale/strategies/base_scaling"
+require "sidekiq_autoscale/strategies/delay_scaling"
+require "sidekiq_autoscale/strategies/linear_scaling"
+require "sidekiq_autoscale/strategies/oldest_job_scaling"
+require "sidekiq_autoscale/strategies/dynamic_latency_scaling"
+require "sidekiq_autoscale/adapters/nil_adapter"
+require "sidekiq_autoscale/adapters/heroku_adapter"
+require "sidekiq_autoscale/adapters/kubernetes_adapter"
+require "sidekiq_autoscale/middleware"
+require "sidekiq_autoscale/config/callbacks"
+require "sidekiq_autoscale/config/shared_configs"
+
+module SidekiqAutoscale
+  class << self
+    include SidekiqAutoscale::Config::SharedConfigs
+
+    def configure
+      yield config
+    end
+  end
+end

data/lib/tasks/sidekiq_autoscale_tasks.rake

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+# desc "Explaining what the task does"
+# task :sidekiq_autoscale do
+#   # Task goes here
+# end

metadata

@@ -0,0 +1,365 @@
+--- !ruby/object:Gem::Specification
+name: sidekiq_autoscale
+version: !ruby/object:Gem::Version
+  version: 0.2.2
+platform: ruby
+authors:
+- Steven Allen
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2022-03-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.8'
+- !ruby/object:Gem::Dependency
+  name: k8s-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.12'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.12'
+- !ruby/object:Gem::Dependency
+  name: platform-api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+- !ruby/object:Gem::Dependency
+  name: railties
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6'
+- !ruby/object:Gem::Dependency
+  name: redlock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.19'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.19'
+- !ruby/object:Gem::Dependency
+  name: awesome_print
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: guard-bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.7'
+- !ruby/object:Gem::Dependency
+  name: mock_redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.50'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.50'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.16'
+- !ruby/object:Gem::Dependency
+  name: simplecov-console
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A simple gem to handle Sidekiq autoscaling.
+email:
+- sallen@tractionguest.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- MIT-LICENSE
+- README.md
+- Rakefile
+- lib/generators/sidekiq_autoscale/install/install_generator.rb
+- lib/generators/sidekiq_autoscale/install/templates/sidekiq_autoscale_initializer_template.template
+- lib/sidekiq_autoscale.rb
+- lib/sidekiq_autoscale/adapters/heroku_adapter.rb
+- lib/sidekiq_autoscale/adapters/kubernetes_adapter.rb
+- lib/sidekiq_autoscale/adapters/nil_adapter.rb
+- lib/sidekiq_autoscale/config/callbacks.rb
+- lib/sidekiq_autoscale/config/shared_configs.rb
+- lib/sidekiq_autoscale/exception.rb
+- lib/sidekiq_autoscale/middleware.rb
+- lib/sidekiq_autoscale/railtie.rb
+- lib/sidekiq_autoscale/sidekiq_interface.rb
+- lib/sidekiq_autoscale/strategies/base_scaling.rb
+- lib/sidekiq_autoscale/strategies/delay_scaling.rb
+- lib/sidekiq_autoscale/strategies/dynamic_latency_scaling.rb
+- lib/sidekiq_autoscale/strategies/linear_scaling.rb
+- lib/sidekiq_autoscale/strategies/oldest_job_scaling.rb
+- lib/sidekiq_autoscale/version.rb
+- lib/tasks/sidekiq_autoscale_tasks.rake
+homepage: https://github.com/tractionguest/sidekiq_autoscaling
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.32
+signing_key: 
+specification_version: 4
+summary: A simple gem to handle Sidekiq autoscaling.
+test_files: []