sidekiq-heroku-autoscale 0.0.1

data/lib/sidekiq/heroku_autoscale/process.rb

@@ -0,0 +1,266 @@
+module Sidekiq
+  module HerokuAutoscale
+
+    class Process
+      WAKE_THROTTLE = PollInterval.new(:wait_for_update!, before_update: 2)
+      SHUTDOWN_POLL = PollInterval.new(:wait_for_shutdown!, before_update: 10)
+
+      attr_reader :app_name, :name, :throttle, :history, :client
+      attr_reader :queue_system, :scale_strategy
+
+      attr_accessor :active_at, :updated_at, :quieted_at
+      attr_accessor :dynos, :quieted_to, :quiet_buffer
+
+      def initialize(
+        name: 'worker',
+        app_name: nil,
+        client: nil,
+        throttle: 10, # 10 seconds
+        history: 3600, # 1 hour
+        quiet_buffer: 10,
+        system: {},
+        scale: {}
+      )
+        @app_name = app_name || name.to_s
+        @name = name.to_s
+        @client = client
+        @queue_system = QueueSystem.new(system)
+        @scale_strategy = ScaleStrategy.new(scale)
+
+        @dynos = 0
+        @active_at = nil
+        @updated_at = nil
+        @quieted_at = nil
+        @quieted_to = nil
+
+        @throttle = throttle
+        @history = history
+        @quiet_buffer = quiet_buffer
+      end
+
+      def status
+        if shutting_down?
+          'stopping'
+        elsif quieting?
+          'quieting'
+        elsif @dynos > 0
+          'running'
+        else
+          'stopped'
+        end
+      end
+
+      # request a throttled update
+      def ping!
+        @active_at = Time.now.utc
+        if ::Sidekiq.server?
+          # submit the process for runscaling (up or down)
+          # the process is polled until shutdown occurs
+          SHUTDOWN_POLL.call(self)
+        else
+          # submits the process for upscaling (wake up)
+          # the process is polled until an update is run
+          WAKE_THROTTLE.call(self)
+        end
+      end
+
+      # checks if the system is downscaling
+      # no other scaling is allowed during a cooling period
+      def quieting?
+        !!(@quieted_to && @quieted_at)
+      end
+
+      def shutting_down?
+        quieting? && @quieted_to.zero?
+      end
+
+      def fulfills_quietdown?
+        !!(@quieted_at && Time.now.utc >= @quieted_at + @quiet_buffer)
+      end
+
+      # check if a probe time is newer than the last update
+      def updated_since_last_activity?
+        !!(@active_at && @updated_at && @updated_at > @active_at)
+      end
+
+      # # check if last update falls within the throttle window
+      def throttled?
+        !!(@updated_at && Time.now.utc < @updated_at + @throttle)
+      end
+
+      # starts a quietdown period in which excess workers are quieted
+      # no formation changes are allowed during a quietdown window.
+      def quietdown(to=0)
+        quiet_to = [0, to].max
+        quiet_at = Time.now.utc
+        unless queue_system.quietdown!(quiet_to)
+          # omit quiet buffer if no workers were actually quieted
+          # allows direct downscaling without buffer delay
+          # (though uptime buffer may still have an effect)
+          quiet_at -= (@quiet_buffer + 1)
+        end
+        set_attributes(quieted_to: quiet_to, quieted_at: quiet_at)
+      end
+
+      # wrapper for throttling the upscale process (client)
+      # polling runs until the next update has been called.
+      def wait_for_update!
+        # resolve (true) when already updated by another process
+        # keep waiting (false) when:
+        # - redundant updates are called within the throttle window
+        # - the system has been fully quieted and must shutdown before upscaling
+        return true if updated_since_last_activity?
+        return false if throttled?
+
+        # first round of checks use local (process-specific) settings
+        # now hit the redis cache and double check settings from other processes
+        sync_attributes
+        return true if updated_since_last_activity?
+        return false if throttled?
+
+        update!
+        true
+      end
+
+      # wrapper for monitoring the downscale process (server)
+      # polling runs until an update returns zero dynos.
+      def wait_for_shutdown!
+        return false if throttled?
+
+        sync_attributes
+        return false if throttled?
+
+        update!.zero?
+      end
+
+      # update the process with live dyno count from Heroku,
+      # and then reassess workload and scale transitions.
+      # this method shouldn't be called directly... just ping! it.
+      def update!(current=nil, target=nil)
+        current ||= fetch_dyno_count
+
+        attrs = { dynos: current, updated_at: Time.now.utc }
+        if current.zero?
+          attrs[:quieted_to] = nil
+          attrs[:quieted_at] = nil
+        end
+        set_attributes(attrs)
+
+        # No changes are allowed while quieting...
+        # the quieted dyno needs to be removed (downscaled)
+        # before making other changes to the formation.
+        unless quieting?
+          # select a new scale target to shoot for
+          # (provides a trajectory, not necessarily a destination)
+          target ||= scale_strategy.call(queue_system)
+
+          # idle
+          if current == target
+            ::Sidekiq.logger.info("IDLE at #{ target } dynos")
+            return current
+
+          # upscale
+          elsif current < target
+            return set_dyno_count!(target)
+
+          # quietdown
+          elsif current > target
+            ::Sidekiq.logger.info("QUIET to #{ current - 1 } dynos")
+            quietdown(current - 1)
+            # do NOT return...
+            # allows downscale conditions to run during the same update
+          end
+        end
+
+        # downscale
+        if quieting? && fulfills_quietdown?
+          return set_dyno_count!(@quieted_to)
+        end
+
+        current
+      end
+
+      # gets a live dyno count from Heroku
+      def fetch_dyno_count
+        if @client
+          @client.formation.list(app_name)
+            .select { |item| item['type'] == name }
+            .map { |item| item['quantity'] }
+            .reduce(0, &:+)
+        else
+          @dynos
+        end
+      rescue StandardError => e
+        ::Sidekiq::HerokuAutoscale.exception_handler.call(e)
+        0
+      end
+
+      # sets the live dyno count on Heroku
+      def set_dyno_count!(count)
+        ::Sidekiq.logger.info("SCALE to #{ count } dynos")
+        @client.formation.update(app_name, name, { quantity: count }) if @client
+        set_attributes(dynos: count, quieted_to: nil, quieted_at: nil, history_at: Time.now.utc)
+        count
+      rescue StandardError => e
+        ::Sidekiq::HerokuAutoscale.exception_handler.call(e)
+        @dynos
+      end
+
+      # sets redis-cached process attributes
+      def set_attributes(attrs)
+        cache = {}
+        prev_dynos = @dynos
+        if attrs.key?(:dynos)
+          cache['dynos'] = @dynos = attrs[:dynos]
+        end
+        if attrs.key?(:quieted_to)
+          cache['quieted_to'] = @quieted_to = attrs[:quieted_to]
+        end
+        if attrs.key?(:quieted_at)
+          @quieted_at = attrs[:quieted_at]
+          cache['quieted_at'] = @quieted_at ? @quieted_at.to_i : nil
+        end
+        if attrs.key?(:updated_at)
+          @updated_at = attrs[:updated_at]
+          cache['updated_at'] = @updated_at ? @updated_at.to_i : nil
+        end
+
+        ::Sidekiq.redis do |c|
+          c.pipelined do
+            # set new keys, delete expired keys
+            del, set = cache.partition { |k, v| v.nil? }
+            c.hmset(cache_key, *set.flatten) if set.any?
+            c.hdel(cache_key, *del.map(&:first)) if del.any?
+
+            if attrs[:history_at]
+              # set a dyno count history marker
+              event_time = (attrs[:history_at].to_f / @throttle).floor * @throttle
+              history_page = (attrs[:history_at].to_f / @history).floor * @history
+              history_key = "#{ cache_key }:#{ history_page }"
+
+              c.hmset(history_key, (event_time - @throttle).to_s, prev_dynos, event_time.to_s, @dynos)
+              c.expire(history_key, @history * 2)
+            end
+          end
+        end
+      end
+
+      # syncs configuration across process instances (dynos)
+      def sync_attributes
+        if cache = ::Sidekiq.redis { |c| c.hgetall(cache_key) }
+          @dynos = cache['dynos'] ? cache['dynos'].to_i : 0
+          @quieted_to = cache['quieted_to'] ? cache['quieted_to'].to_i : nil
+          @quieted_at = cache['quieted_at'] ? Time.at(cache['quieted_at'].to_i).utc : nil
+          @updated_at = cache['updated_at'] ? Time.at(cache['updated_at'].to_i).utc : nil
+          return true
+        end
+        false
+      end
+
+      def cache_key
+        [self.class.name.gsub('::', '/').downcase, app_name, name].join(':')
+      end
+    end
+
+  end
+end

data/lib/sidekiq/heroku_autoscale/queue_system.rb

@@ -0,0 +1,102 @@
+require 'sidekiq/api'
+
+module Sidekiq
+  module HerokuAutoscale
+
+    class QueueSystem
+      ALL_QUEUES = '*'.freeze
+
+      attr_accessor :watch_queues, :include_retrying, :include_scheduled
+
+      def initialize(watch_queues: ALL_QUEUES, include_retrying: true, include_scheduled: true)
+        @watch_queues = [watch_queues].flatten.uniq
+        @include_retrying = include_retrying
+        @include_scheduled = include_scheduled
+      end
+
+      def all_queues?
+        @watch_queues.first == ALL_QUEUES
+      end
+
+      # number of dynos (process instances) running sidekiq
+      # this may include one-or-more instances of one-or-more heroku process types
+      # (though they should all be one process type if setup validation was observed)
+      def dynos
+        sidekiq_processes.size
+      end
+
+      # number of worker threads currently running sidekiq jobs
+      # counts all queue-specific threads across all dynos (process instances)
+      def threads
+        # work => { 'queue' => name, 'run_at' => timestamp, 'payload' => msg }
+        worker_set = ::Sidekiq::Workers.new.to_a
+        worker_set = worker_set.select { |pid, tid, work| watch_queues.include?(work['queue']) } unless all_queues?
+        worker_set.length
+      end
+
+      # number of jobs sitting in the active work queue
+      def enqueued
+        counts = all_queues? ? sidekiq_queues.values : sidekiq_queues.slice(*watch_queues).values
+        counts.map(&:to_i).reduce(&:+) || 0
+      end
+
+      # number of jobs in the scheduled set
+      def scheduled
+        return 0 unless @include_scheduled
+        count_jobs(::Sidekiq::ScheduledSet.new)
+      end
+
+      # number of jobs in the retry set
+      def retrying
+        return 0 unless @include_retrying
+        count_jobs(::Sidekiq::RetrySet.new)
+      end
+
+      def total_work
+        enqueued + scheduled + retrying + threads
+      end
+
+      def has_work?
+        total_work > 0
+      end
+
+      # When scaling down workers, heroku stops the one with the highest number...
+      # from https://stackoverflow.com/questions/25215334/scale-down-specific-heroku-worker-dynos
+      def quietdown!(scale)
+        quieted = false
+        # processes have hostnames formatted as "worker.1", "worker.2", "sidekiq.1", etc...
+        # this groups processes by type, then sorts by number, and then quiets beyond scale.
+        sidekiq_processes.group_by { |p| p['hostname'].split('.').first }.each_pair do |type, group|
+          # there should only ever be a single group here (assuming setup validations were observed)
+          group.sort_by { |p| p['hostname'].split('.').last.to_i }.each_with_index do |process, index|
+            if index + 1 > scale && !process.stopping?
+              process.quiet!
+              quieted = true
+            end
+          end
+        end
+
+        quieted
+      end
+
+      def sidekiq_queues
+        ::Sidekiq::Stats.new.queues
+      end
+
+      def sidekiq_processes
+        process_set = ::Sidekiq::ProcessSet.new
+        # select all processes with queues that intersect watched queues
+        process_set = process_set.select { |p| (p['queues'] & @watch_queues).any? } unless all_queues?
+        process_set
+      end
+
+    private
+
+      def count_jobs(job_set)
+        return job_set.size if all_queues?
+        job_set.count { |j| watch_queues.include?(j.queue) }
+      end
+    end
+
+  end
+end

data/lib/sidekiq/heroku_autoscale/scale_strategy.rb

@@ -0,0 +1,53 @@
+module Sidekiq
+  module HerokuAutoscale
+
+    class ScaleStrategy
+      attr_accessor :mode, :max_dynos, :workers_per_dyno, :min_factor
+
+      def initialize(mode: :binary, max_dynos: 1, workers_per_dyno: 25, min_factor: 0)
+        @mode = mode
+        @max_dynos = max_dynos
+        @workers_per_dyno = workers_per_dyno
+        @min_factor = min_factor
+      end
+
+      def call(sys)
+        case @mode.to_s
+        when 'linear'
+          linear(sys)
+        else
+          binary(sys)
+        end
+      end
+
+      def binary(sys)
+        sys.has_work? ? @max_dynos : 0
+      end
+
+      def linear(sys)
+        # total capacity of max workers
+        total_capacity = (@max_dynos * @workers_per_dyno).to_f
+
+        # min capacity required to scale first worker
+        min_capacity = [0, @min_factor].max.to_f * @workers_per_dyno
+
+        # min percentage of total capacity
+        min_capacity_percentage = min_capacity / total_capacity
+        requested_capacity_percentage = sys.total_work / total_capacity
+
+        # Scale requested capacity taking into account the minimum required
+        scale_factor = (requested_capacity_percentage - min_capacity_percentage) / (total_capacity - min_capacity_percentage)
+        scale_factor = 0 if scale_factor.nan? # Handle DIVZERO
+        scaled_capacity_percentage = scale_factor * total_capacity
+
+        # don't scale down past number of currently engaged workers,
+        # and don't scale up past maximum dynos
+        ideal_dynos = ([0, scaled_capacity_percentage].max * @max_dynos).ceil
+        minimum_dynos = [sys.dynos, ideal_dynos].max
+        maximum_dynos = [minimum_dynos, @max_dynos].min
+        [minimum_dynos, maximum_dynos].min
+      end
+    end
+
+  end
+end

data/lib/sidekiq/heroku_autoscale/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module HerokuAutoscale
+    VERSION = '0.0.1'
+  end
+end

data/lib/sidekiq/heroku_autoscale/web.rb

@@ -0,0 +1,7 @@
+require 'sidekiq/heroku_autoscale/process'
+require 'sidekiq/heroku_autoscale/web_extension'
+
+if defined?(::Sidekiq::Web)
+  ::Sidekiq::Web.register(::Sidekiq::HerokuAutoscale::WebExtension)
+  ::Sidekiq::Web.tabs["Dynos"] = "dynos"
+end

data/lib/sidekiq/heroku_autoscale/web/inactive.erb

@@ -0,0 +1,4 @@
+<div class= "dashboard clearfix">
+  <h3>Dynos</h3>
+  <p>Heroku Autoscale is not initialized.</p>
+</div>

data/lib/sidekiq/heroku_autoscale/web/index.erb

@@ -0,0 +1,38 @@
+<div class= "dashboard clearfix">
+  <h3>Heorku Dynos
+    <span class="beacon" id="beacon">
+      <span class="ring"></span>
+      <span class="dot"></span>
+    </span>
+  </h3>
+</div>
+
+<div class="row chart">
+  <div id="history" data-update-url="<%= root_path %>stats" data-dynos-url="<%= root_path %>dynos/stats"></div>
+  <div id="history-legend"></div>
+  <script id="history-data" type="text/json">
+    <%= JSON.generate(@dyno_stats) %>
+  </script>
+</div>
+
+<h5>Process types</h5>
+<div class="table_container">
+  <table class="processes table table-hover table-bordered table-striped table-white">
+    <thead>
+      <th>Name</th>
+      <th>Updated at</th>
+      <th>Status</th>
+      <th>Dynos</th>
+    </thead>
+    <% @dyno_stats.each_pair do |key, stats| %>
+      <tr>
+        <td><%= key %></td>
+        <td id="<%= key %>-updated"><%= stats[:updated] %></td>
+        <td id="<%= key %>-status"><%= stats[:status] %></td>
+        <td id="<%= key %>-dynos"><%= stats[:dynos] %></td>
+      </tr>
+    <% end %>
+  </table>
+</div>
+
+<script type="text/javascript" src="<%= root_path %>dynos/index.js"></script>