autoscaler 0.3.0 → 0.4.0

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 0.4.0
+
+- Experimental: The default scaling logic is contained in BinaryScalingStrategy.  A strategy object can be passed instead of timeout to the server middleware.
+
 ## 0.3.0
 
 - Downscale method changed from busy-waiting workers to a separate monitor process

data/README.md

@@ -42,6 +42,10 @@ Install the middleware in your `Sidekiq.configure_` blocks
 - The retry and schedule lists are considered - if you schedule a long-running task, the process will not scale-down.
 - If background jobs trigger jobs in other scaled processes, please note you'll need `config.client_middleware` in your `Sidekiq.configure_server` block in order to scale-up.
 
+## Experimental
+
+You can pass a scaling strategy object instead of the timeout to the server middleware.  The object (or lambda) should respond to `#call(system, idle_time)` and return the desired number of workers.  See `lib/autoscaler/binary_scaling_strategy.rb` for an example.
+
 ## Tests
 
 The project is setup to run RSpec with Guard.  It expects a redis instance on a custom port, which is started by the Guardfile.

data/lib/autoscaler/binary_scaling_strategy.rb

@@ -0,0 +1,26 @@
+module Autoscaler
+  # Strategies determine the target number of workers
+  # The default strategy has a single worker when there is anything, or shuts it down.
+  class BinaryScalingStrategy
+    #@params [integer] active_workers number of workers when in the active state.
+    def initialize(active_workers = 1)
+      @active_workers = active_workers
+    end
+
+    # @param [QueueSystem] system interface to the queuing system
+    # @param [Numeric] event_idle_time number of seconds since a job related event
+    # @return [Integer] target number of workers
+    def call(system, event_idle_time)
+      if active?(system)
+        @active_workers
+      else
+        0
+      end
+    end
+
+    private
+    def active?(system)
+      system.queued > 0 || system.scheduled > 0 || system.retrying > 0 || system.workers > 0
+    end
+  end
+end

data/lib/autoscaler/sidekiq/celluloid_monitor.rb

@@ -7,27 +7,28 @@ module Autoscaler
       include Celluloid
 
       # @param [scaler] scaler object that actually performs scaling operations (e.g. {HerokuScaler})
-      # @param [Numeric] timeout number of seconds to wait before shutdown
-      # @param [System] system interface to the queuing system that provides `pending_work?`
-      def initialize(scaler, timeout, system)
+      # @param [Strategy] strategy object that decides the target number of workers (e.g. {BinaryScalingStrategy})
+      # @param [System] system interface to the queuing system for use by the strategy
+      def initialize(scaler, strategy, system)
         @scaler = scaler
-        @timeout = timeout
-        @poll = [timeout/4.0, 0.5].max
+        @strategy = strategy
         @system = system
         @running = false
       end
 
-      # Mostly sleep until there has been no activity for the timeout
-      def wait_for_downscale
+      # Periodically update the desired number of workers
+      # @param [Numeric] interval polling interval, mostly for testing
+      def wait_for_downscale(interval = 15)
         once do
           active_now!
 
-          while active? || time_left?
-            sleep(@poll)
-            update_activity
-          end
+          workers = @scaler.workers
 
-          @scaler.workers = 0
+          begin
+            sleep(interval)
+            target_workers = @strategy.call(@system, idle_time)
+            workers = @scaler.workers = target_workers unless workers == target_workers
+          end while workers > 0
         end
       end
 
@@ -42,22 +43,13 @@ module Autoscaler
       end
 
       private
-      attr_reader :system
-
-      def active?
-        system.pending_work? || system.working?
-      end
-
-      def update_activity
-        active_now! if active?
-      end
 
       def active_now!
         @activity = Time.now
       end
 
-      def time_left?
-        (Time.now - @activity) < @timeout
+      def idle_time
+        Time.now - @activity
       end
 
       def once

data/lib/autoscaler/sidekiq/client.rb

@@ -11,7 +11,11 @@ module Autoscaler
 
       # Sidekiq middleware api method
       def call(worker_class, item, queue)
-        @scalers[queue] && @scalers[queue].workers = 1
+        scaler = @scalers[queue]
+        if scaler && scaler.workers < 1
+          scaler.workers = 1
+        end
+
         yield
       end
     end

data/lib/autoscaler/sidekiq/entire_queue_system.rb

@@ -0,0 +1,44 @@
+require 'sidekiq/api'
+
+module Autoscaler
+  module Sidekiq
+    # Interface to to interrogate the queuing system
+    # Includes every queue
+    class EntireQueueSystem
+      # @return [Integer] number of worker actively engaged
+      def workers
+        ::Sidekiq::Workers.new.size
+      end
+
+      # @return [Integer] amount work ready to go
+      def queued
+        sidekiq_queues.values.map(&:to_i).reduce(&:+)
+      end
+
+      # @return [Integer] amount of work scheduled for some time in the future
+      def scheduled
+        count_sorted_set("schedule")
+      end
+
+      # @return [Integer] amount of work still being retried
+      def retrying
+        count_sorted_set("retry")
+      end
+
+      # @return [Array[String]]
+      def queue_names
+        sidekiq_queues.keys
+      end
+
+      private
+
+      def sidekiq_queues
+        ::Sidekiq::Stats.new.queues
+      end
+
+      def count_sorted_set(sorted_set)
+        ::Sidekiq::SortedSet.new(sorted_set).count
+      end
+    end
+  end
+end

data/lib/autoscaler/sidekiq/monitor_middleware_adapter.rb

@@ -1,5 +1,7 @@
 require 'autoscaler/sidekiq/queue_system'
 require 'autoscaler/sidekiq/celluloid_monitor'
+require 'autoscaler/binary_scaling_strategy'
+require 'autoscaler/delayed_shutdown'
 
 module Autoscaler
   module Sidekiq
@@ -7,12 +9,14 @@ module Autoscaler
     # Starts the monitor and notifies it of job events that may occur while it's sleeping
     class MonitorMiddlewareAdapter
       # @param [scaler] scaler object that actually performs scaling operations (e.g. {HerokuScaler})
-      # @param [Numeric] timeout number of seconds to wait before shutdown
+      # @param [Strategy,Numeric] timeout strategy object that determines target workers, or a timeout to be passed to {DelayedShutdown}+{BinaryScalingStrategy}
       # @param [Array[String]] specified_queues list of queues to monitor to determine if there is work left.  Defaults to all sidekiq queues.
       def initialize(scaler, timeout, specified_queues = nil)
-        system = QueueSystem.new(specified_queues)
         unless monitor
-          CelluloidMonitor.supervise_as(:autoscaler_monitor, scaler, timeout, system)
+          CelluloidMonitor.supervise_as(:autoscaler_monitor,
+                                        scaler,
+                                        strategy(timeout),
+                                        QueueSystem.new(specified_queues))
         end
       end
 
@@ -28,6 +32,14 @@ module Autoscaler
       def monitor
         Celluloid::Actor[:autoscaler_monitor]
       end
+
+      def strategy(timeout)
+        if timeout.respond_to?(:call)
+          timeout
+        else
+          DelayedShutdown.new(BinaryScalingStrategy.new, timeout)
+        end
+      end
     end
   end
 end

data/lib/autoscaler/sidekiq/queue_system.rb

@@ -1,72 +1,20 @@
 require 'sidekiq/api'
+require 'autoscaler/sidekiq/specified_queue_system'
+require 'autoscaler/sidekiq/entire_queue_system'
 
 module Autoscaler
   module Sidekiq
     # Interface to to interrogate the queuing system
-    class QueueSystem
+    # convenience constructor for SpecifiedQueueSystem and EntireQueueSystem
+    module QueueSystem
       # @param [Array[String]] specified_queues list of queues to monitor to determine if there is work left.  Defaults to all sidekiq queues.
-      def initialize(specified_queues = nil)
-        @specified_queues = specified_queues
-      end
-
-      # @return [boolean] whether there is queued work - does not include work currently in progress
-      def pending_work?
-        refresh_sidekiq_queues!
-        queued_work? || scheduled_work? || retry_work?
-      end
-
-      # @return [boolean] whether there are workers actively running
-      def working?
-        workers > 0
-      end
-
-      # @return [Array[String]]
-      def queue_names
-        (@specified_queues || sidekiq_queues.keys)
-      end
-
-      private
-      def queued_work?
-        queue_names.any? {|name| sidekiq_queues[name].to_i > 0 }
-      end
-
-      def scheduled_work?
-        empty_sorted_set?("schedule")
-      end
-
-      def retry_work?
-        empty_sorted_set?("retry")
-      end
-
-      attr_reader :sidekiq_queues
-
-      def refresh_sidekiq_queues!
-        @sidekiq_queues = ::Sidekiq::Stats.new.queues
-      end
-
-      def empty_sorted_set?(sorted_set)
-        ss = ::Sidekiq::SortedSet.new(sorted_set)
-        ss.any? { |job| queue_names.include?(job.queue) }
-      end
-
-      def workers
-        if @specified_queues
-          specified_workers
+      def self.new(specified_queues = nil)
+        if specified_queues
+          SpecifiedQueueSystem.new(specified_queues)
         else
-          total_workers
+          EntireQueueSystem.new
         end
       end
-
-      def total_workers
-        ::Sidekiq::Workers.new.size
-      end
-
-      def specified_workers
-        refresh_sidekiq_queues!
-        ::Sidekiq::Workers.new.count {|name, work|
-          queue_names.include?(work['queue'])
-        }
-      end
     end
   end
 end

data/lib/autoscaler/sidekiq/sleep_wait_server.rb

@@ -37,7 +37,7 @@ module Autoscaler
       attr_reader :activity
 
       def pending_work?
-        system.pending_work?
+        system.queued > 0 || system.scheduled > 0 || system.retrying > 0
       end
 
       def working!(queue)

data/lib/autoscaler/sidekiq/specified_queue_system.rb

@@ -0,0 +1,49 @@
+require 'sidekiq/api'
+
+module Autoscaler
+  module Sidekiq
+    # Interface to to interrogate the queuing system
+    # Includes only the queues provided to the constructor
+    class SpecifiedQueueSystem
+      # @param [Array[String]] specified_queues list of queues to monitor to determine if there is work left.  Defaults to all sidekiq queues.
+      def initialize(specified_queues)
+        @queue_names = specified_queues
+      end
+
+      # @return [Integer] number of worker actively engaged
+      def workers
+        ::Sidekiq::Workers.new.count {|name, work|
+          queue_names.include?(work['queue'])
+        }
+      end
+
+      # @return [Integer] amount work ready to go
+      def queued
+        queue_names.map {|name| sidekiq_queues[name].to_i}.reduce(&:+)
+      end
+
+      # @return [Integer] amount of work scheduled for some time in the future
+      def scheduled
+        count_sorted_set("schedule")
+      end
+
+      # @return [Integer] amount of work still being retried
+      def retrying
+        count_sorted_set("retry")
+      end
+
+      # @return [Array[String]]
+      attr_reader :queue_names
+
+      private
+      def sidekiq_queues
+        ::Sidekiq::Stats.new.queues
+      end
+
+      def count_sorted_set(sorted_set)
+        ss = ::Sidekiq::SortedSet.new(sorted_set)
+        ss.count { |job| queue_names.include?(job.queue) }
+      end
+    end
+  end
+end

data/lib/autoscaler/version.rb

@@ -1,4 +1,4 @@
 module Autoscaler
   # version number
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

data/spec/autoscaler/binary_scaling_strategy_spec.rb

@@ -0,0 +1,19 @@
+require 'spec_helper'
+require 'test_system'
+require 'autoscaler/binary_scaling_strategy'
+
+describe Autoscaler::BinaryScalingStrategy do
+  let(:cut) {Autoscaler::BinaryScalingStrategy}
+
+  it "scales with no work" do
+    system = TestSystem.new(0)
+    strategy = cut.new
+    strategy.call(system, 1).should == 0
+  end
+
+  it "does not scale with pending work" do
+    system = TestSystem.new(1)
+    strategy = cut.new(2)
+    strategy.call(system, 1).should == 2
+  end
+end

data/spec/autoscaler/sidekiq/celluloid_monitor_spec.rb

@@ -1,16 +1,8 @@
 require 'spec_helper'
+require 'test_system'
 require 'autoscaler/sidekiq/celluloid_monitor'
 require 'timeout'
 
-class TestSystem
-  def initialize(pending)
-    @pending = pending
-  end
-
-  def working?; false; end
-  def pending_work?; @pending; end
-end
-
 describe Autoscaler::Sidekiq::CelluloidMonitor do
   before do
     @redis = Sidekiq.redis = REDIS
@@ -21,17 +13,17 @@ describe Autoscaler::Sidekiq::CelluloidMonitor do
   let(:scaler) {TestScaler.new(1)}
 
   it "scales with no work" do
-    system = TestSystem.new(false)
-    manager = cut.new(scaler, 0, system)
-    Timeout.timeout(1) { manager.wait_for_downscale }
+    system = TestSystem.new(0)
+    manager = cut.new(scaler, lambda{|s,t| 0}, system)
+    Timeout.timeout(1) { manager.wait_for_downscale(0.5) }
     scaler.workers.should == 0
     manager.terminate
   end
 
   it "does not scale with pending work" do
-    system = TestSystem.new(true)
-    manager = cut.new(scaler, 0, system)
-    expect {Timeout.timeout(1) { manager.wait_for_downscale }}.to raise_error Timeout::Error
+    system = TestSystem.new(1)
+    manager = cut.new(scaler, lambda{|s,t| 1}, system)
+    expect {Timeout.timeout(1) { manager.wait_for_downscale(0.5) }}.to raise_error Timeout::Error
     scaler.workers.should == 1
     manager.terminate
   end

data/spec/autoscaler/sidekiq/entire_queue_system_spec.rb

@@ -0,0 +1,61 @@
+require 'spec_helper'
+require 'autoscaler/sidekiq/entire_queue_system'
+
+describe Autoscaler::Sidekiq::EntireQueueSystem do
+  let(:cut) {Autoscaler::Sidekiq::EntireQueueSystem}
+
+  before do
+    @redis = Sidekiq.redis = REDIS
+    Sidekiq.redis {|c| c.flushdb }
+  end
+
+  def with_work_in_set(queue, set)
+    payload = Sidekiq.dump_json('queue' => queue)
+    Sidekiq.redis { |c| c.zadd(set, (Time.now.to_f + 30.to_f).to_s, payload)}
+  end
+
+  def with_scheduled_work_in(queue)
+    with_work_in_set(queue, 'schedule')
+  end
+
+  def with_retry_work_in(queue)
+    with_work_in_set(queue, 'retry')
+  end
+
+  subject {cut.new}
+
+  it {subject.queue_names.should == []}
+  it {subject.workers.should == 0}
+
+  describe 'no queued work' do
+    it "with no work" do
+      subject.stub(:sidekiq_queues).and_return({'queue' => 0, 'another_queue' => 0})
+      subject.queued.should == 0
+    end
+
+    it "with no scheduled work" do
+      subject.scheduled.should == 0
+    end
+
+    it "with no retry work" do
+      subject.retrying.should == 0
+    end
+  end
+
+  describe 'with queued work' do
+    it "with enqueued work" do
+      subject.stub(:sidekiq_queues).and_return({'queue' => 1})
+      subject.queued.should == 1
+    end
+
+    it "with schedule work" do
+      with_scheduled_work_in('queue')
+      subject.scheduled.should == 1
+    end
+
+    it "with retry work" do
+      with_retry_work_in('queue')
+      subject.retrying.should == 1
+    end
+  end
+end

data/spec/autoscaler/sidekiq/{queue_system_spec.rb → specified_queue_system_spec.rb}

@@ -1,8 +1,8 @@
 require 'spec_helper'
-require 'autoscaler/sidekiq/queue_system'
+require 'autoscaler/sidekiq/specified_queue_system'
 
-describe Autoscaler::Sidekiq::QueueSystem do
-  let(:cut) {Autoscaler::Sidekiq::QueueSystem}
+describe Autoscaler::Sidekiq::SpecifiedQueueSystem do
+  let(:cut) {Autoscaler::Sidekiq::SpecifiedQueueSystem}
 
   before do
     @redis = Sidekiq.redis = REDIS
@@ -25,39 +25,39 @@ describe Autoscaler::Sidekiq::QueueSystem do
   subject {cut.new(['queue'])}
 
   it {subject.queue_names.should == ['queue']}
-  it {subject.working?.should be_false}
+  it {subject.workers.should == 0}
 
-  describe 'no pending work' do
+  describe 'no queued work' do
     it "with no work" do
       subject.stub(:sidekiq_queues).and_return({'queue' => 0, 'another_queue' => 1})
-      subject.should_not be_pending_work
+      subject.queued.should == 0
     end
 
     it "with scheduled work in another queue" do
       with_scheduled_work_in('another_queue')
-      subject.should_not be_pending_work
+      subject.scheduled.should == 0
     end
 
     it "with retry work in another queue" do
       with_retry_work_in('another_queue')
-      subject.should_not be_pending_work
+      subject.retrying.should == 0
     end
   end
 
-  describe 'with pending work' do
+  describe 'with queued work' do
     it "with enqueued work" do
       subject.stub(:sidekiq_queues).and_return({'queue' => 1})
-      subject.should be_pending_work
+      subject.queued.should == 1
     end
 
     it "with schedule work" do
       with_scheduled_work_in('queue')
-      subject.should be_pending_work
+      subject.scheduled.should == 1
     end
 
     it "with retry work" do
       with_retry_work_in('queue')
-      subject.should be_pending_work
+      subject.retrying.should == 1
     end
   end
 end

data/spec/test_system.rb

@@ -0,0 +1,10 @@
+class TestSystem
+  def initialize(pending)
+    @pending = pending
+  end
+
+  def workers; 0; end
+  def queued; @pending; end
+  def scheduled; 0; end
+  def retrying; 0; end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: autoscaler
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
   prerelease: 
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-05-18 00:00:00.000000000 Z
+date: 2013-06-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -132,13 +132,16 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- lib/autoscaler/binary_scaling_strategy.rb
 - lib/autoscaler/heroku_scaler.rb
 - lib/autoscaler/sidekiq/activity.rb
 - lib/autoscaler/sidekiq/celluloid_monitor.rb
 - lib/autoscaler/sidekiq/client.rb
+- lib/autoscaler/sidekiq/entire_queue_system.rb
 - lib/autoscaler/sidekiq/monitor_middleware_adapter.rb
 - lib/autoscaler/sidekiq/queue_system.rb
 - lib/autoscaler/sidekiq/sleep_wait_server.rb
+- lib/autoscaler/sidekiq/specified_queue_system.rb
 - lib/autoscaler/sidekiq.rb
 - lib/autoscaler/stub_scaler.rb
 - lib/autoscaler/version.rb
@@ -148,15 +151,18 @@ files:
 - examples/complex.rb
 - examples/simple.rb
 - Guardfile
+- spec/autoscaler/binary_scaling_strategy_spec.rb
 - spec/autoscaler/heroku_scaler_spec.rb
 - spec/autoscaler/sidekiq/activity_spec.rb
 - spec/autoscaler/sidekiq/celluloid_monitor_spec.rb
 - spec/autoscaler/sidekiq/client_spec.rb
+- spec/autoscaler/sidekiq/entire_queue_system_spec.rb
 - spec/autoscaler/sidekiq/monitor_middleware_adapter_spec.rb
-- spec/autoscaler/sidekiq/queue_system_spec.rb
 - spec/autoscaler/sidekiq/sleep_wait_server_spec.rb
+- spec/autoscaler/sidekiq/specified_queue_system_spec.rb
 - spec/redis_test.conf
 - spec/spec_helper.rb
+- spec/test_system.rb
 homepage: ''
 licenses: []
 post_install_message: 
@@ -183,13 +189,16 @@ specification_version: 3
 summary: Start/stop Sidekiq workers on Heroku
 test_files:
 - Guardfile
+- spec/autoscaler/binary_scaling_strategy_spec.rb
 - spec/autoscaler/heroku_scaler_spec.rb
 - spec/autoscaler/sidekiq/activity_spec.rb
 - spec/autoscaler/sidekiq/celluloid_monitor_spec.rb
 - spec/autoscaler/sidekiq/client_spec.rb
+- spec/autoscaler/sidekiq/entire_queue_system_spec.rb
 - spec/autoscaler/sidekiq/monitor_middleware_adapter_spec.rb
-- spec/autoscaler/sidekiq/queue_system_spec.rb
 - spec/autoscaler/sidekiq/sleep_wait_server_spec.rb
+- spec/autoscaler/sidekiq/specified_queue_system_spec.rb
 - spec/redis_test.conf
 - spec/spec_helper.rb
+- spec/test_system.rb
 has_rdoc: