autoscaler 0.1.0 → 0.2.0

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## 0.2.0
+
+- Raise minimum Sidekiq version to 2.6.1 to take advantage of Stats API
+- Inspect scheduled and retry sets to see if they match `specified_queues`
+- Testing: Refactor server middleware tests
+
 ## 0.1.0
 
 - The `retry` and `scheduled` queues are now considered for shutdown

data/README.md

@@ -58,7 +58,7 @@ Justin Love, [@wondible](http://twitter.com/wondible), [https://github.com/Justi
 
 Ported to Heroku-Api by Fix Peña, [https://github.com/fixr](https://github.com/fixr)
 
-Retry/schedule queues by Matt Anderson [https://github.com/tonkapark](https://github.com/tonkapark)
+Retry/schedule sets by Matt Anderson [https://github.com/tonkapark](https://github.com/tonkapark) and Thibaud Guillaume-Gentil [https://github.com/jilion](https://github.com/jilion)
 
 ## Licence
 

data/examples/complex.rb

@@ -1,4 +1,3 @@
-require 'securerandom' # bug in Sidekiq as of 2.2.1
 require 'sidekiq'
 require 'autoscaler/sidekiq'
 require 'autoscaler/heroku_scaler'

data/examples/simple.rb

@@ -1,4 +1,3 @@
-require 'securerandom' # bug in Sidekiq as of 2.2.1
 require 'sidekiq'
 require 'autoscaler/sidekiq'
 require 'autoscaler/heroku_scaler'

data/lib/autoscaler/sidekiq.rb

@@ -1,4 +1,3 @@
-require 'securerandom' # bug in Sidekiq as of 2.2.1
 require 'sidekiq'
 
 module Autoscaler
@@ -42,35 +41,41 @@ module Autoscaler
       end
 
       private
-      def queues
-        @specified_queues || registered_queues
+      def refresh_sidekiq_queues!
+        @sidekiq_queues = ::Sidekiq::Stats.new.queues
       end
 
-      def registered_queues
-        ::Sidekiq.redis { |x| x.smembers('queues') }
+      attr_reader :sidekiq_queues
+
+      def queue_names
+        (@specified_queues || sidekiq_queues.keys)
       end
 
-      def empty?(name)
-        ::Sidekiq.redis { |conn| conn.llen("queue:#{name}") == 0 }
+      def queued_work?
+        queue_names.any? {|name| sidekiq_queues[name].to_i > 0 }
       end
-      
+
       def scheduled_work?
-        ::Sidekiq.redis { |c| c.zcard("schedule") > 0 } 
+        empty_sorted_set?("schedule")
       end
-      
+
       def retry_work?
-        ::Sidekiq.redis { |c| c.zcard("retry") > 0 } 
-      end  
+        empty_sorted_set?("retry")
+      end
+
+      def empty_sorted_set?(sorted_set)
+        ss = ::Sidekiq::SortedSet.new(sorted_set)
+        ss.any? { |job| queue_names.include?(job.queue) }
+      end
 
       def pending_work?
-        queues.any? {|q| !empty?(q)}
+        refresh_sidekiq_queues!
+        queued_work? || scheduled_work? || retry_work?
       end
 
       def wait_for_task_or_scale
         loop do
           return if pending_work?
-          return if scheduled_work?
-          return if retry_work?
           return @scaler.workers = 0 if idle?
           sleep(0.5)
         end

data/lib/autoscaler/version.rb

@@ -1,4 +1,4 @@
 module Autoscaler
   # version number
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/spec/autoscaler/sidekiq_spec.rb

@@ -14,60 +14,110 @@ describe Autoscaler::Sidekiq do
     @redis = Sidekiq.redis = REDIS
     Sidekiq.redis {|c| c.flushdb }
   end
-    
+
   let(:scaler) do
     Scaler.new(workers)
   end
 
   describe Autoscaler::Sidekiq::Client do
     let(:cut) {Autoscaler::Sidekiq::Client}
-    let(:sa) {cut.new('queue' => scaler)}
+    let(:client) {cut.new('queue' => scaler)}
     let(:workers) {0}
 
     describe 'scales' do
-      before {sa.call(Class, {}, 'queue') {}}
-      subject {scaler.workers}
-      it {should == 1}
+      before {client.call(Class, {}, 'queue') {}}
+      it {scaler.workers.should == 1}
     end
 
     describe 'yields' do
-      it {sa.call(Class, {}, 'queue') {:foo}.should == :foo}
+      it {client.call(Class, {}, 'queue') {:foo}.should == :foo}
     end
   end
 
   describe Autoscaler::Sidekiq::Server do
     let(:cut) {Autoscaler::Sidekiq::Server}
-    let(:sa) {cut.new(scaler, 0)}
+    let(:server) {cut.new(scaler, 0, ['queue'])}
     let(:workers) {1}
 
+    def with_work_in_set(queue, set)
+      payload = Sidekiq.dump_json('queue' => queue)
+      Sidekiq.redis { |c| c.zadd(set, (Time.now.to_f + 30.to_f).to_s, payload)}
+    end
+
+    def with_scheduled_work_in(queue)
+      with_work_in_set(queue, 'schedule')
+    end
+
+    def with_retry_work_in(queue)
+      with_work_in_set(queue, 'retry')
+    end
+
+    def when_run
+      server.call(Object.new, {}, 'queue') {}
+    end
+
+    def self.when_run_should_scale
+      it('should downscale') do
+        when_run
+        scaler.workers.should == 0
+      end
+    end
+
+    def self.when_run_should_not_scale
+      it('should not downscale') do
+        when_run
+        scaler.workers.should == 1
+      end
+    end
+
     describe 'scales' do
-      context "with only enqueued work" do
-        before{sa.call(Object.new, {}, 'queue') {}}
-        subject {scaler.workers}
-        it {should == 0}
+      context "with no work" do
+        before do
+          server.stub(:sidekiq_queues).and_return({'queue' => 0, 'another_queue' => 1})
+        end
+        when_run_should_scale
+      end
+
+      context "with scheduled work in another queue" do
+        before do
+          with_scheduled_work_in('another_queue')
+        end
+        when_run_should_scale
+      end
+
+      context "with retry work in another queue" do
+        before do
+          with_retry_work_in('another_queue')
+        end
+        when_run_should_scale
+      end
+    end
+
+    describe 'does not scale' do
+      context "with enqueued work" do
+        before do
+          server.stub(:sidekiq_queues).and_return({'queue' => 1})
+        end
+        when_run_should_not_scale
       end
-      
+
       context "with schedule work" do
-        before do 
-          Sidekiq.redis { |c| c.zadd('schedule', (Time.now.to_f + 30.to_f).to_s, '{}' )}
-          sa.call(Object.new, {}, 'queue') {}
+        before do
+          with_scheduled_work_in('queue')
         end
-        subject {scaler.workers}
-        it {should == 1}
+        when_run_should_not_scale
       end
-      
+
       context "with retry work" do
         before do
-          Sidekiq.redis { |c| c.zadd('retry', (Time.now.to_f + 30.to_f).to_s, '{}' )}
-          sa.call(Object.new, {}, 'queue') {}
-        end  
-        subject {scaler.workers}      
-        it {should == 1}
+          with_retry_work_in('queue')
+        end
+        when_run_should_not_scale
       end
     end
-    
+
     describe 'yields' do
-      it {sa.call(Object.new, {}, 'queue') {:foo}.should == :foo}
-    end    
+      it {server.call(Object.new, {}, 'queue') {:foo}.should == :foo}
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: autoscaler
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -10,24 +10,30 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-01-20 00:00:00.000000000 Z
+date: 2013-01-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ~>
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.6.1
+    - - <
       - !ruby/object:Gem::Version
-        version: '2.2'
+        version: '3.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ~>
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.6.1
+    - - <
       - !ruby/object:Gem::Version
-        version: '2.2'
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: heroku-api
   requirement: !ruby/object:Gem::Requirement