autoscaler 0.10.0 → 0.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3d1fdf02eef4bd9e9ed8ca36ec7a88b0e298ead8
-  data.tar.gz: fd639d4797b01dc4aded9d0d11dc0d386d24f39e
+  metadata.gz: c7c64a64f5b36a7615b3fc8587d78ef94f608649
+  data.tar.gz: bd6f84ada4beb4cd9a6d933809da089aeeee6ae7
 SHA512:
-  metadata.gz: 43c6d465b6bbbb1978d9ace6e5adc60db0a4c4aac72b206706f10397121dafc9ad2c59edd1a02395a10df8b01927abf445977be997d60bf0d19e172ed24aa85b
-  data.tar.gz: 27fdfa89b504de1b90a735929e96d9c96e7fd0f84e2b8d80ea05a2ec58d786af71f726109649dfa220b7d9a5cd038f34dc591d28a9218587c073a2a6b8fb9961
+  metadata.gz: e5ae3abf03d4ce9dc2c1bf7d7510f4d4c1b132e5f180dac7882630bd9a0bcf36ba43baf43338896b980909215ee0cadd66687ea438561aa880f9ec84f702cedb
+  data.tar.gz: 6f26a32336472672ee927d55578c4ce1da108163d58ed4c7494b92ea5c3e2c027d6013d42dab4e05d151b1a7772f1d82e717e3f7ce80ad6794204842219c13bf

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## 0.11.0
+
+- Replace celluloid monitor with thread based middleware for Sidekiq 4.0
+- Tests no longer use use redis namespace
+
 ## 0.10.0
 
 - Require Sidekiq 3.5

data/README.md

@@ -12,7 +12,7 @@ Tested on Ruby 2.1.7 and Heroku Cedar stack.
 
 ## Getting Started
 
-This gem uses the [Heroku Platform-Api](https://github.com/heroku/platform-api.rb) gem, which requires an OAuth token from Heroku.  It will also need the heroku app name.  By default, these are specified through environment variables.  You can also pass them to `HerokuPlatformScaler` explicitly.
+This gem uses the [Heroku Platform-Api](https://github.com/heroku/platform-api) gem, which requires an OAuth token from Heroku.  It will also need the heroku app name.  By default, these are specified through environment variables.  You can also pass them to `HerokuPlatformScaler` explicitly.
 
     HEROKU_ACCESS_TOKEN=.....
     HEROKU_APP=....
@@ -77,6 +77,7 @@ Justin Love, [@wondible](http://twitter.com/wondible), [https://github.com/Justi
 ### Contributors
 
 - Benjamin Kudria [https://github.com/bkudria](https://github.com/bkudria)
+- claudiofullscreen [https://github.com/claudiofullscreen](https://github.com/claudiofullscreen)
 - Fix Peña [https://github.com/fixr](https://github.com/fixr)
 - Gabriel Givigier Guimarães [https://github.com/givigier](https://github.com/givigier)
 - Matt Anderson [https://github.com/tonkapark](https://github.com/tonkapark)

data/lib/autoscaler/sidekiq.rb

@@ -1,11 +1,11 @@
 require 'autoscaler/sidekiq/client'
-require 'autoscaler/sidekiq/monitor_middleware_adapter'
+require 'autoscaler/sidekiq/thread_server'
 
 module Autoscaler
   # namespace module for Sidekiq middlewares
   module Sidekiq
     # Sidekiq server middleware
     # Performs scale-down when the queue is empty
-    Server = MonitorMiddlewareAdapter
+    Server = ThreadServer
   end
 end

data/lib/autoscaler/sidekiq/thread_server.rb

@@ -0,0 +1,90 @@
+require 'autoscaler/sidekiq/queue_system'
+require 'autoscaler/binary_scaling_strategy'
+require 'autoscaler/delayed_shutdown'
+require 'thread'
+
+module Autoscaler
+  module Sidekiq
+    # Sidekiq server middleware
+    # spawns a thread to monitor the sidekiq server for scale-down
+    class ThreadServer
+      # @param [scaler] scaler object that actually performs scaling operations (e.g. {HerokuPlatformScaler})
+      # @param [Strategy,Numeric] timeout strategy object that determines target workers, or a timeout  in seconds to be passed to {DelayedShutdown}+{BinaryScalingStrategy}
+      # @param [Array[String]] specified_queues list of queues to monitor to determine if there is work left.  Defaults to all sidekiq queues.
+      def initialize(scaler, timeout, specified_queues = nil)
+        @scaler = scaler
+        @strategy = strategy(timeout)
+        @system = QueueSystem.new(specified_queues)
+        @mutex = Mutex.new
+        @done = false
+      end
+
+      # Sidekiq middleware api entry point
+      def call(worker, msg, queue, _ = nil)
+        yield
+      ensure
+        active_now!
+        wait_for_downscale
+      end
+
+      # Start the monitoring thread if it's not running
+      def wait_for_downscale
+        @thread ||= Thread.new do
+          begin
+            run
+          rescue
+            @thread = nil
+          end
+        end
+      end
+
+      # Thread core loop
+      # Periodically update the desired number of workers
+      # @param [Numeric] interval polling interval, mostly for testing
+      def run(interval = 15)
+        active_now!
+
+        workers = :unknown
+
+        begin
+          sleep(interval)
+          target_workers = @strategy.call(@system, idle_time)
+          workers = @scaler.workers = target_workers unless workers == target_workers
+        end while !@done && workers > 0
+        ::Sidekiq::ProcessSet.new.each(&:quiet!)
+      end
+
+      # Shut down the thread, pause until complete
+      def terminate
+        @done = true
+        if @thread
+          t = @thread
+          @thread = nil
+          t.value
+        end
+      end
+
+      private
+
+      def active_now!
+        @mutex.synchronize do
+          @activity = Time.now
+        end
+      end
+
+      def idle_time
+        @mutex.synchronize do
+          Time.now - @activity
+        end
+      end
+
+      def strategy(timeout)
+        if timeout.respond_to?(:call)
+          timeout
+        else
+          DelayedShutdown.new(BinaryScalingStrategy.new, timeout)
+        end
+      end
+    end
+  end
+end

data/lib/autoscaler/version.rb

@@ -1,4 +1,4 @@
 module Autoscaler
   # version number
-  VERSION = "0.10.0"
+  VERSION = "0.11.0"
 end

data/spec/autoscaler/counter_cache_redis_spec.rb

@@ -41,7 +41,7 @@ describe Autoscaler::CounterCacheRedis do
   end
 
   it 'passed a plain connection' do
-    connection = Redis.connect(:url => 'redis://localhost:9736', :namespace => 'autoscaler')
+    connection = Redis.connect(:url => 'redis://localhost:9736')
     cache = cut.new connection
     cache.counter = 5
     expect(cache.counter).to eq 5

data/spec/autoscaler/sidekiq/activity_spec.rb

@@ -26,7 +26,7 @@ describe Autoscaler::Sidekiq::Activity do
   end
 
   it 'passed a plain connection' do
-    connection = Redis.connect(:url => 'redis://localhost:9736', :namespace => 'autoscaler')
+    connection = Redis.connect(:url => 'redis://localhost:9736')
     activity = cut.new(5, connection)
     activity.working!('queue')
     expect(activity).to_not be_idle(['queue'])

data/spec/autoscaler/sidekiq/thread_server_spec.rb

@@ -0,0 +1,44 @@
+require 'spec_helper'
+require 'autoscaler/sidekiq/thread_server'
+require 'timeout'
+
+describe Autoscaler::Sidekiq::ThreadServer do
+  before do
+    @redis = Sidekiq.redis = REDIS
+    Sidekiq.redis {|c| c.flushdb }
+  end
+
+  let(:cut) {Autoscaler::Sidekiq::ThreadServer}
+  let(:scaler) {TestScaler.new(1)}
+
+  describe "thread core" do
+    it "scales with no work" do
+      server = cut.new(scaler, lambda{|s,t| 0})
+      Timeout.timeout(1) { server.run(0.5) }
+      expect(scaler.workers).to eq 0
+      server.terminate
+    end
+
+    it "does not scale with pending work" do
+      server = cut.new(scaler, lambda{|s,t| 1})
+      expect {Timeout.timeout(1) { server.run(0.5) }}.to raise_error Timeout::Error
+      expect(scaler.workers).to eq 1
+      server.terminate
+    end
+
+    it "will downscale with initial workers zero" do
+      scaler = TestScaler.new(0)
+      server = cut.new(scaler, lambda{|s,t| 0})
+      Timeout.timeout(1) { server.run(0.5) }
+      expect(scaler.workers).to eq 0
+      server.terminate
+    end
+  end
+
+  describe "Middleware interface" do
+    let(:server) {cut.new(scaler, 0, ['queue'])}
+
+    it('yields') {expect(server.call(Object.new, {}, 'queue') {:foo}).to eq :foo}
+    it('yields with a redis pool') {expect(server.call(Object.new, {}, 'queue', Sidekiq.method(:redis)) {:foo}).to eq :foo}
+  end
+end

data/spec/spec_helper.rb

@@ -1,6 +1,6 @@
 require 'rspec/its'
 require 'sidekiq'
-REDIS = Sidekiq::RedisConnection.create(:url => 'redis://localhost:9736', :namespace => 'autoscaler')
+REDIS = Sidekiq::RedisConnection.create(:url => 'redis://localhost:9736')
 
 RSpec.configure do |config|
   config.mock_with :rspec

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: autoscaler
 version: !ruby/object:Gem::Version
-  version: 0.10.0
+  version: 0.11.0
 platform: ruby
 authors:
 - Justin Love
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-27 00:00:00.000000000 Z
+date: 2015-11-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -17,28 +17,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.5.1
+        version: '4.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.5.1
-- !ruby/object:Gem::Dependency
-  name: celluloid
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.17.2
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.17.2
+        version: '4.0'
 - !ruby/object:Gem::Dependency
   name: heroku-api
   requirement: !ruby/object:Gem::Requirement
@@ -161,13 +147,12 @@ files:
 - lib/autoscaler/linear_scaling_strategy.rb
 - lib/autoscaler/sidekiq.rb
 - lib/autoscaler/sidekiq/activity.rb
-- lib/autoscaler/sidekiq/celluloid_monitor.rb
 - lib/autoscaler/sidekiq/client.rb
 - lib/autoscaler/sidekiq/entire_queue_system.rb
-- lib/autoscaler/sidekiq/monitor_middleware_adapter.rb
 - lib/autoscaler/sidekiq/queue_system.rb
 - lib/autoscaler/sidekiq/sleep_wait_server.rb
 - lib/autoscaler/sidekiq/specified_queue_system.rb
+- lib/autoscaler/sidekiq/thread_server.rb
 - lib/autoscaler/stub_scaler.rb
 - lib/autoscaler/version.rb
 - spec/autoscaler/binary_scaling_strategy_spec.rb
@@ -179,12 +164,11 @@ files:
 - spec/autoscaler/ignore_scheduled_and_retrying_spec.rb
 - spec/autoscaler/linear_scaling_strategy_spec.rb
 - spec/autoscaler/sidekiq/activity_spec.rb
-- spec/autoscaler/sidekiq/celluloid_monitor_spec.rb
 - spec/autoscaler/sidekiq/client_spec.rb
 - spec/autoscaler/sidekiq/entire_queue_system_spec.rb
-- spec/autoscaler/sidekiq/monitor_middleware_adapter_spec.rb
 - spec/autoscaler/sidekiq/sleep_wait_server_spec.rb
 - spec/autoscaler/sidekiq/specified_queue_system_spec.rb
+- spec/autoscaler/sidekiq/thread_server_spec.rb
 - spec/spec_helper.rb
 - spec/test_system.rb
 homepage: https://github.com/JustinLove/autoscaler
@@ -221,12 +205,11 @@ test_files:
 - spec/autoscaler/ignore_scheduled_and_retrying_spec.rb
 - spec/autoscaler/linear_scaling_strategy_spec.rb
 - spec/autoscaler/sidekiq/activity_spec.rb
-- spec/autoscaler/sidekiq/celluloid_monitor_spec.rb
 - spec/autoscaler/sidekiq/client_spec.rb
 - spec/autoscaler/sidekiq/entire_queue_system_spec.rb
-- spec/autoscaler/sidekiq/monitor_middleware_adapter_spec.rb
 - spec/autoscaler/sidekiq/sleep_wait_server_spec.rb
 - spec/autoscaler/sidekiq/specified_queue_system_spec.rb
+- spec/autoscaler/sidekiq/thread_server_spec.rb
 - spec/spec_helper.rb
 - spec/test_system.rb
 has_rdoc: 

data/lib/autoscaler/sidekiq/celluloid_monitor.rb

@@ -1,68 +0,0 @@
-require 'celluloid/current'
-
-module Autoscaler
-  module Sidekiq
-    # Actor to monitor the sidekiq server for scale-down
-    class CelluloidMonitor
-      include Celluloid
-
-      # @param [scaler] scaler object that actually performs scaling operations (e.g. {HerokuPlatformScaler})
-      # @param [Strategy] strategy object that decides the target number of workers (e.g. {BinaryScalingStrategy})
-      # @param [System] system interface to the queuing system for use by the strategy
-      def initialize(scaler, strategy, system)
-        @scaler = scaler
-        @strategy = strategy
-        @system = system
-        @running = false
-      end
-
-      # Periodically update the desired number of workers
-      # @param [Numeric] interval polling interval, mostly for testing
-      def wait_for_downscale(interval = 15)
-        once do
-          active_now!
-
-          workers = :unknown
-
-          begin
-            sleep(interval)
-            target_workers = @strategy.call(@system, idle_time)
-            workers = @scaler.workers = target_workers unless workers == target_workers
-          end while workers > 0
-          ::Sidekiq::ProcessSet.new.each(&:quiet!)
-        end
-      end
-
-      # Notify the monitor that a job is starting
-      def starting_job
-      end
-
-      # Notify the monitor that a job has finished
-      def finished_job
-        active_now!
-        async.wait_for_downscale
-      end
-
-      private
-
-      def active_now!
-        @activity = Time.now
-      end
-
-      def idle_time
-        Time.now - @activity
-      end
-
-      def once
-        return if @running
-
-        begin
-          @running = true
-          yield
-        ensure
-          @running = false
-        end
-      end
-    end
-  end
-end

data/lib/autoscaler/sidekiq/monitor_middleware_adapter.rb

@@ -1,48 +0,0 @@
-require 'autoscaler/sidekiq/queue_system'
-require 'autoscaler/sidekiq/celluloid_monitor'
-require 'autoscaler/binary_scaling_strategy'
-require 'autoscaler/delayed_shutdown'
-
-module Autoscaler
-  module Sidekiq
-    # Shim to the existing autoscaler interface
-    # Starts the monitor and notifies it of job events that may occur while it's sleeping
-    class MonitorMiddlewareAdapter
-      # @param [scaler] scaler object that actually performs scaling operations (e.g. {HerokuPlatformScaler})
-      # @param [Strategy,Numeric] timeout strategy object that determines target workers, or a timeout  in seconds to be passed to {DelayedShutdown}+{BinaryScalingStrategy}
-      # @param [Array[String]] specified_queues list of queues to monitor to determine if there is work left.  Defaults to all sidekiq queues.
-      def initialize(scaler, timeout, specified_queues = nil)
-        unless monitor
-          CelluloidMonitor.supervise :as => :autoscaler_monitor,
-                                     :args => [
-                                        scaler,
-                                        strategy(timeout),
-                                        QueueSystem.new(specified_queues),
-                                     ]
-        end
-      end
-
-      # Sidekiq middleware api entry point
-      def call(worker, msg, queue, _ = nil)
-        monitor.async.starting_job
-        yield
-      ensure
-        # monitor might have gone, e.g. if Sidekiq has received SIGTERM
-        monitor.async.finished_job if monitor
-      end
-
-      private
-      def monitor
-        Celluloid::Actor[:autoscaler_monitor]
-      end
-
-      def strategy(timeout)
-        if timeout.respond_to?(:call)
-          timeout
-        else
-          DelayedShutdown.new(BinaryScalingStrategy.new, timeout)
-        end
-      end
-    end
-  end
-end

data/spec/autoscaler/sidekiq/celluloid_monitor_spec.rb

@@ -1,39 +0,0 @@
-require 'spec_helper'
-require 'test_system'
-require 'autoscaler/sidekiq/celluloid_monitor'
-require 'timeout'
-
-describe Autoscaler::Sidekiq::CelluloidMonitor do
-  before do
-    @redis = Sidekiq.redis = REDIS
-    Sidekiq.redis {|c| c.flushdb }
-  end
-
-  let(:cut) {Autoscaler::Sidekiq::CelluloidMonitor}
-  let(:scaler) {TestScaler.new(1)}
-
-  it "scales with no work" do
-    system = TestSystem.new(0)
-    manager = cut.new(scaler, lambda{|s,t| 0}, system)
-    Timeout.timeout(1) { manager.wait_for_downscale(0.5) }
-    expect(scaler.workers).to eq 0
-    manager.terminate
-  end
-
-  it "does not scale with pending work" do
-    system = TestSystem.new(1)
-    manager = cut.new(scaler, lambda{|s,t| 1}, system)
-    expect {Timeout.timeout(1) { manager.wait_for_downscale(0.5) }}.to raise_error Timeout::Error
-    expect(scaler.workers).to eq 1
-    manager.terminate
-  end
-
-  it "will downscale with initial workers zero" do
-    system = TestSystem.new(0)
-    scaler = TestScaler.new(0)
-    manager = cut.new(scaler, lambda{|s,t| 0}, system)
-    Timeout.timeout(1) { manager.wait_for_downscale(0.5) }
-    expect(scaler.workers).to eq 0
-    manager.terminate
-  end
-end

data/spec/autoscaler/sidekiq/monitor_middleware_adapter_spec.rb

@@ -1,16 +0,0 @@
-require 'spec_helper'
-require 'autoscaler/sidekiq/monitor_middleware_adapter'
-
-describe Autoscaler::Sidekiq::MonitorMiddlewareAdapter do
-  before do
-    @redis = Sidekiq.redis = REDIS
-    Sidekiq.redis {|c| c.flushdb }
-  end
-
-  let(:cut) {Autoscaler::Sidekiq::MonitorMiddlewareAdapter}
-  let(:scaler) {TestScaler.new(1)}
-  let(:server) {cut.new(scaler, 0, ['queue'])}
-
-  it('yields') {expect(server.call(Object.new, {}, 'queue') {:foo}).to eq :foo}
-  it('yields with a redis pool') {expect(server.call(Object.new, {}, 'queue', Sidekiq.method(:redis)) {:foo}).to eq :foo}
-end