autoscale 0.9.1

data/lib/autoscaler/ignore_scheduled_and_retrying.rb

@@ -0,0 +1,13 @@
+module Autoscaler
+  class IgnoreScheduledAndRetrying
+    def initialize(strategy)
+      @strategy = strategy
+    end
+
+    def call(system, event_idle_time)
+      system.define_singleton_method(:scheduled) { 0 }
+      system.define_singleton_method(:retrying)  { 0 }
+      @strategy.call(system, event_idle_time)
+    end
+  end
+end

data/lib/autoscaler/linear_scaling_strategy.rb

@@ -0,0 +1,39 @@
+module Autoscaler
+  # Strategies determine the target number of workers
+  # This strategy sets the number of workers to be proportional to the number of enqueued jobs.
+  class LinearScalingStrategy
+    #@param [integer] max_workers     maximum number of workers to spin up.
+    #@param [integer] worker_capacity the amount of jobs one worker can handle
+    #@param [float]   min_factor      minimum work required to scale, as percentage of worker_capacity
+    def initialize(max_workers = 1, worker_capacity = 25, min_factor = 0)
+      @max_workers             = max_workers # max # of workers we can scale to
+      @total_capacity          = (@max_workers * worker_capacity).to_f # total capacity of max workers
+      min_capacity             = [0, min_factor].max.to_f * worker_capacity # min capacity required to scale first worker
+      @min_capacity_percentage = min_capacity / @total_capacity # min percentage of total capacity
+    end
+
+    # @param [QueueSystem] system interface to the queuing system
+    # @param [Numeric] event_idle_time number of seconds since a job related event
+    # @return [Integer] target number of workers
+    def call(system, event_idle_time)
+      requested_capacity_percentage = total_work(system) / @total_capacity
+
+      # Scale requested capacity taking into account the minimum required
+      scale_factor = (requested_capacity_percentage - @min_capacity_percentage) / (@total_capacity - @min_capacity_percentage)
+      scale_factor = 0 if scale_factor.nan? # Handle DIVZERO
+
+      scaled_capacity_percentage = scale_factor * @total_capacity
+
+      ideal_workers = ([0, scaled_capacity_percentage].max * @max_workers).ceil
+      min_workers   = [system.workers, ideal_workers].max  # Don't scale down past number of currently engaged workers
+      max_workers   = [min_workers,  @max_workers].min     # Don't scale up past number of max workers
+
+      return [min_workers, max_workers].min
+    end
+
+    private
+    def total_work(system)
+      system.queued + system.scheduled + system.retrying
+    end
+  end
+end

data/lib/autoscaler/sidekiq.rb

@@ -0,0 +1,11 @@
+require 'autoscaler/sidekiq/client'
+require 'autoscaler/sidekiq/monitor_middleware_adapter'
+
+module Autoscaler
+  # namespace module for Sidekiq middlewares
+  module Sidekiq
+    # Sidekiq server middleware
+    # Performs scale-down when the queue is empty
+    Server = MonitorMiddlewareAdapter
+  end
+end

data/lib/autoscaler/sidekiq/activity.rb

@@ -0,0 +1,62 @@
+require 'sidekiq'
+
+module Autoscaler
+  module Sidekiq
+    # Tracks activity timeouts using Sidekiq's redis connection
+    class Activity
+      # @param [Numeric] timeout number of seconds to wait before shutdown
+      def initialize(timeout, redis = ::Sidekiq.method(:redis))
+        @timeout = timeout
+        @redis   = redis
+      end
+
+      # Record that a queue has activity
+      # @param [String] queue
+      def working!(queue)
+        active_at queue, Time.now
+      end
+
+      # Record that a queue is idle and timed out - mostly for test support
+      # @param [String] queue
+      def idle!(queue)
+        active_at queue, Time.now - timeout*2
+      end
+
+      # Have the watched queues timed out?
+      # @param [Array[String]] queues list of queues to monitor to determine if there is work left
+      # @return [boolean]
+      def idle?(queues)
+        idle_time(queues) > timeout
+      end
+
+      private
+      attr_reader :timeout
+
+      def idle_time(queues)
+        t = last_activity(queues)
+        return 0 unless t
+        Time.now - Time.parse(t)
+      end
+
+      def last_activity(queues)
+        redis {|c|
+          queues.map {|q| c.get('background_activity:'+q)}.compact.max
+        }
+      end
+
+      def active_at(queue, time)
+        redis {|c| c.set('background_activity:'+queue, time)}
+      end
+
+      def redis(&block)
+        if @redis.respond_to?(:call)
+          @redis.call(&block)
+        elsif @redis.respond_to?(:with)
+          @redis.with(&block)
+        else
+          block.call(@redis)
+        end
+      end
+    end
+  end
+end

data/lib/autoscaler/sidekiq/celluloid_monitor.rb

@@ -0,0 +1,67 @@
+require 'celluloid'
+
+module Autoscaler
+  module Sidekiq
+    # Actor to monitor the sidekiq server for scale-down
+    class CelluloidMonitor
+      include Celluloid
+
+      # @param [scaler] scaler object that actually performs scaling operations (e.g. {HerokuScaler})
+      # @param [Strategy] strategy object that decides the target number of workers (e.g. {BinaryScalingStrategy})
+      # @param [System] system interface to the queuing system for use by the strategy
+      def initialize(scaler, strategy, system)
+        @scaler = scaler
+        @strategy = strategy
+        @system = system
+        @running = false
+      end
+
+      # Periodically update the desired number of workers
+      # @param [Numeric] interval polling interval, mostly for testing
+      def wait_for_downscale(interval = 15)
+        once do
+          active_now!
+
+          workers = :unknown
+
+          begin
+            sleep(interval)
+            target_workers = @strategy.call(@system, idle_time)
+            workers = @scaler.workers = target_workers unless workers == target_workers
+          end while workers > 0
+        end
+      end
+
+      # Notify the monitor that a job is starting
+      def starting_job
+      end
+
+      # Notify the monitor that a job has finished
+      def finished_job
+        active_now!
+        async.wait_for_downscale
+      end
+
+      private
+
+      def active_now!
+        @activity = Time.now
+      end
+
+      def idle_time
+        Time.now - @activity
+      end
+
+      def once
+        return if @running
+
+        begin
+          @running = true
+          yield
+        ensure
+          @running = false
+        end
+      end
+    end
+  end
+end

data/lib/autoscaler/sidekiq/client.rb

@@ -0,0 +1,50 @@
+require 'autoscaler/binary_scaling_strategy'
+require 'autoscaler/sidekiq/specified_queue_system'
+
+module Autoscaler
+  module Sidekiq
+    # Sidekiq client middleware
+    # Performs scale-up when items are queued and there are no workers running
+    class Client
+      # @param [Hash] scalers map of queue(String) => scaler (e.g. {HerokuScaler}).
+      #   Which scaler to use for each sidekiq queue
+      def initialize(scalers)
+        @scalers = scalers
+      end
+
+      # Sidekiq middleware api method
+      def call(worker_class, item, queue, _ = nil)
+        result = yield
+
+        scaler = @scalers[queue]
+        if scaler && scaler.workers < 1
+          scaler.workers = 1
+        end
+
+        result
+      end
+
+      # Check for interrupted or scheduled work on startup.
+      # Typically you need to construct your own instance just
+      # to call this method, but see add_to_chain.
+      # @param [Strategy] strategy object that determines target workers
+      # @yieldparam [String] queue mostly for testing
+      # @yieldreturn [QueueSystem] mostly for testing
+      def set_initial_workers(strategy = nil, &system_factory)
+        strategy ||= BinaryScalingStrategy.new
+        system_factory ||= lambda {|queue| SpecifiedQueueSystem.new([queue])} 
+        @scalers.each do |queue, scaler|
+          scaler.workers = strategy.call(system_factory.call(queue), 0)
+        end
+      end
+
+      # Convenience method to avoid having to name the class and parameter
+      # twice when calling set_initial_workers
+      # @return [Client] an instance of Client for set_initial_workers
+      def self.add_to_chain(chain, scalers)
+        chain.add self, scalers
+        new(scalers)
+      end
+    end
+  end
+end

data/lib/autoscaler/sidekiq/entire_queue_system.rb

@@ -0,0 +1,41 @@
+require 'sidekiq/api'
+
+module Autoscaler
+  module Sidekiq
+    # Interface to to interrogate the queuing system
+    # Includes every queue
+    class EntireQueueSystem
+      # @return [Integer] number of workers actively engaged
+      def workers
+        ::Sidekiq::Workers.new.map {|pid, _, _| pid}.uniq.size
+        # #size may be out-of-date.
+      end
+
+      # @return [Integer] amount work ready to go
+      def queued
+        sidekiq_queues.values.map(&:to_i).reduce(&:+) || 0
+      end
+
+      # @return [Integer] amount of work scheduled for some time in the future
+      def scheduled
+        ::Sidekiq::ScheduledSet.new.size
+      end
+
+      # @return [Integer] amount of work still being retried
+      def retrying
+        ::Sidekiq::RetrySet.new.size
+      end
+
+      # @return [Array[String]]
+      def queue_names
+        sidekiq_queues.keys
+      end
+
+      private
+
+      def sidekiq_queues
+        ::Sidekiq::Stats.new.queues
+      end
+    end
+  end
+end

data/lib/autoscaler/sidekiq/monitor_middleware_adapter.rb

@@ -0,0 +1,46 @@
+require 'autoscaler/sidekiq/queue_system'
+require 'autoscaler/sidekiq/celluloid_monitor'
+require 'autoscaler/binary_scaling_strategy'
+require 'autoscaler/delayed_shutdown'
+
+module Autoscaler
+  module Sidekiq
+    # Shim to the existing autoscaler interface
+    # Starts the monitor and notifies it of job events that may occur while it's sleeping
+    class MonitorMiddlewareAdapter
+      # @param [scaler] scaler object that actually performs scaling operations (e.g. {HerokuScaler})
+      # @param [Strategy,Numeric] timeout strategy object that determines target workers, or a timeout  in seconds to be passed to {DelayedShutdown}+{BinaryScalingStrategy}
+      # @param [Array[String]] specified_queues list of queues to monitor to determine if there is work left.  Defaults to all sidekiq queues.
+      def initialize(scaler, timeout, specified_queues = nil)
+        unless monitor
+          CelluloidMonitor.supervise_as(:autoscaler_monitor,
+                                        scaler,
+                                        strategy(timeout),
+                                        QueueSystem.new(specified_queues))
+        end
+      end
+
+      # Sidekiq middleware api entry point
+      def call(worker, msg, queue, _ = nil)
+        monitor.async.starting_job
+        yield
+      ensure
+        # monitor might have gone, e.g. if Sidekiq has received SIGTERM
+        monitor.async.finished_job if monitor
+      end
+
+      private
+      def monitor
+        Celluloid::Actor[:autoscaler_monitor]
+      end
+
+      def strategy(timeout)
+        if timeout.respond_to?(:call)
+          timeout
+        else
+          DelayedShutdown.new(BinaryScalingStrategy.new, timeout)
+        end
+      end
+    end
+  end
+end

data/lib/autoscaler/sidekiq/queue_system.rb

@@ -0,0 +1,20 @@
+require 'sidekiq/api'
+require 'autoscaler/sidekiq/specified_queue_system'
+require 'autoscaler/sidekiq/entire_queue_system'
+
+module Autoscaler
+  module Sidekiq
+    # Interface to to interrogate the queuing system
+    # convenience constructor for SpecifiedQueueSystem and EntireQueueSystem
+    module QueueSystem
+      # @param [Array[String]] specified_queues list of queues to monitor to determine if there is work left.  Defaults to all sidekiq queues.
+      def self.new(specified_queues = nil)
+        if specified_queues
+          SpecifiedQueueSystem.new(specified_queues)
+        else
+          EntireQueueSystem.new
+        end
+      end
+    end
+  end
+end

data/lib/autoscaler/sidekiq/sleep_wait_server.rb

@@ -0,0 +1,51 @@
+require 'autoscaler/sidekiq/queue_system'
+require 'autoscaler/sidekiq/activity'
+
+module Autoscaler
+  module Sidekiq
+    # Sidekiq server middleware
+    # Performs scale-down when the queue is empty
+    class SleepWaitServer
+      # @param [scaler] scaler object that actually performs scaling operations (e.g. {HerokuScaler})
+      # @param [Numeric] timeout number of seconds to wait before shutdown
+      # @param [Array[String]] specified_queues list of queues to monitor to determine if there is work left.  Defaults to all sidekiq queues.
+      def initialize(scaler, timeout, specified_queues = nil)
+        @scaler  = scaler
+        @timeout = timeout
+        @system  = QueueSystem.new(specified_queues)
+      end
+
+      # Sidekiq middleware api entry point
+      def call(worker, msg, queue, redis = ::Sidekiq.method(:redis))
+        working!(queue, redis)
+        yield
+      ensure
+        working!(queue, redis)
+        wait_for_task_or_scale(redis)
+      end
+
+      private
+      def wait_for_task_or_scale(redis)
+        loop do
+          return if pending_work?
+          return @scaler.workers = 0 if idle?(redis)
+          sleep(0.5)
+        end
+      end
+
+      attr_reader :system
+
+      def pending_work?
+        system.queued > 0 || system.scheduled > 0 || system.retrying > 0
+      end
+
+      def working!(queue, redis)
+        Activity.new(@timeout, redis).working!(queue)
+      end
+
+      def idle?(redis)
+        Activity.new(@timeout, redis).idle?(system.queue_names)
+      end
+    end
+  end
+end

data/lib/autoscaler/sidekiq/specified_queue_system.rb

@@ -0,0 +1,48 @@
+require 'sidekiq/api'
+
+module Autoscaler
+  module Sidekiq
+    # Interface to to interrogate the queuing system
+    # Includes only the queues provided to the constructor
+    class SpecifiedQueueSystem
+      # @param [Array[String]] specified_queues list of queues to monitor to determine if there is work left.  Defaults to all sidekiq queues.
+      def initialize(specified_queues)
+        @queue_names = specified_queues
+      end
+
+      # @return [Integer] number of workers actively engaged
+      def workers
+        ::Sidekiq::Workers.new.select {|_, _, work|
+          queue_names.include?(work['queue'])
+        }.map {|pid, _, _| pid}.uniq.size
+      end
+
+      # @return [Integer] amount work ready to go
+      def queued
+        queue_names.map {|name| sidekiq_queues[name].to_i}.reduce(&:+)
+      end
+
+      # @return [Integer] amount of work scheduled for some time in the future
+      def scheduled
+        count_set(::Sidekiq::ScheduledSet.new)
+      end
+
+      # @return [Integer] amount of work still being retried
+      def retrying
+        count_set(::Sidekiq::RetrySet.new)
+      end
+
+      # @return [Array[String]]
+      attr_reader :queue_names
+
+      private
+      def sidekiq_queues
+        ::Sidekiq::Stats.new.queues
+      end
+
+      def count_set(set)
+        set.count { |job| queue_names.include?(job.queue) }
+      end
+    end
+  end
+end