autoscaler 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/CHANGELOG.md CHANGED
@@ -1,5 +1,9 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.4.0
4
+
5
+ - Experimental: The default scaling logic is contained in BinaryScalingStrategy. A strategy object can be passed instead of timeout to the server middleware.
6
+
3
7
  ## 0.3.0
4
8
 
5
9
  - Downscale method changed from busy-waiting workers to a separate monitor process
data/README.md CHANGED
@@ -42,6 +42,10 @@ Install the middleware in your `Sidekiq.configure_` blocks
42
42
  - The retry and schedule lists are considered - if you schedule a long-running task, the process will not scale-down.
43
43
  - If background jobs trigger jobs in other scaled processes, please note you'll need `config.client_middleware` in your `Sidekiq.configure_server` block in order to scale-up.
44
44
 
45
+ ## Experimental
46
+
47
+ You can pass a scaling strategy object instead of the timeout to the server middleware. The object (or lambda) should respond to `#call(system, idle_time)` and return the desired number of workers. See `lib/autoscaler/binary_scaling_strategy.rb` for an example.
48
+
45
49
  ## Tests
46
50
 
47
51
  The project is setup to run RSpec with Guard. It expects a redis instance on a custom port, which is started by the Guardfile.
@@ -0,0 +1,26 @@
1
+ module Autoscaler
2
+ # Strategies determine the target number of workers
3
+ # The default strategy has a single worker when there is anything, or shuts it down.
4
+ class BinaryScalingStrategy
5
+ #@params [integer] active_workers number of workers when in the active state.
6
+ def initialize(active_workers = 1)
7
+ @active_workers = active_workers
8
+ end
9
+
10
+ # @param [QueueSystem] system interface to the queuing system
11
+ # @param [Numeric] event_idle_time number of seconds since a job related event
12
+ # @return [Integer] target number of workers
13
+ def call(system, event_idle_time)
14
+ if active?(system)
15
+ @active_workers
16
+ else
17
+ 0
18
+ end
19
+ end
20
+
21
+ private
22
+ def active?(system)
23
+ system.queued > 0 || system.scheduled > 0 || system.retrying > 0 || system.workers > 0
24
+ end
25
+ end
26
+ end
@@ -7,27 +7,28 @@ module Autoscaler
7
7
  include Celluloid
8
8
 
9
9
  # @param [scaler] scaler object that actually performs scaling operations (e.g. {HerokuScaler})
10
- # @param [Numeric] timeout number of seconds to wait before shutdown
11
- # @param [System] system interface to the queuing system that provides `pending_work?`
12
- def initialize(scaler, timeout, system)
10
+ # @param [Strategy] strategy object that decides the target number of workers (e.g. {BinaryScalingStrategy})
11
+ # @param [System] system interface to the queuing system for use by the strategy
12
+ def initialize(scaler, strategy, system)
13
13
  @scaler = scaler
14
- @timeout = timeout
15
- @poll = [timeout/4.0, 0.5].max
14
+ @strategy = strategy
16
15
  @system = system
17
16
  @running = false
18
17
  end
19
18
 
20
- # Mostly sleep until there has been no activity for the timeout
21
- def wait_for_downscale
19
+ # Periodically update the desired number of workers
20
+ # @param [Numeric] interval polling interval, mostly for testing
21
+ def wait_for_downscale(interval = 15)
22
22
  once do
23
23
  active_now!
24
24
 
25
- while active? || time_left?
26
- sleep(@poll)
27
- update_activity
28
- end
25
+ workers = @scaler.workers
29
26
 
30
- @scaler.workers = 0
27
+ begin
28
+ sleep(interval)
29
+ target_workers = @strategy.call(@system, idle_time)
30
+ workers = @scaler.workers = target_workers unless workers == target_workers
31
+ end while workers > 0
31
32
  end
32
33
  end
33
34
 
@@ -42,22 +43,13 @@ module Autoscaler
42
43
  end
43
44
 
44
45
  private
45
- attr_reader :system
46
-
47
- def active?
48
- system.pending_work? || system.working?
49
- end
50
-
51
- def update_activity
52
- active_now! if active?
53
- end
54
46
 
55
47
  def active_now!
56
48
  @activity = Time.now
57
49
  end
58
50
 
59
- def time_left?
60
- (Time.now - @activity) < @timeout
51
+ def idle_time
52
+ Time.now - @activity
61
53
  end
62
54
 
63
55
  def once
@@ -11,7 +11,11 @@ module Autoscaler
11
11
 
12
12
  # Sidekiq middleware api method
13
13
  def call(worker_class, item, queue)
14
- @scalers[queue] && @scalers[queue].workers = 1
14
+ scaler = @scalers[queue]
15
+ if scaler && scaler.workers < 1
16
+ scaler.workers = 1
17
+ end
18
+
15
19
  yield
16
20
  end
17
21
  end
@@ -0,0 +1,44 @@
1
+ require 'sidekiq/api'
2
+
3
+ module Autoscaler
4
+ module Sidekiq
5
+ # Interface to to interrogate the queuing system
6
+ # Includes every queue
7
+ class EntireQueueSystem
8
+ # @return [Integer] number of worker actively engaged
9
+ def workers
10
+ ::Sidekiq::Workers.new.size
11
+ end
12
+
13
+ # @return [Integer] amount work ready to go
14
+ def queued
15
+ sidekiq_queues.values.map(&:to_i).reduce(&:+)
16
+ end
17
+
18
+ # @return [Integer] amount of work scheduled for some time in the future
19
+ def scheduled
20
+ count_sorted_set("schedule")
21
+ end
22
+
23
+ # @return [Integer] amount of work still being retried
24
+ def retrying
25
+ count_sorted_set("retry")
26
+ end
27
+
28
+ # @return [Array[String]]
29
+ def queue_names
30
+ sidekiq_queues.keys
31
+ end
32
+
33
+ private
34
+
35
+ def sidekiq_queues
36
+ ::Sidekiq::Stats.new.queues
37
+ end
38
+
39
+ def count_sorted_set(sorted_set)
40
+ ::Sidekiq::SortedSet.new(sorted_set).count
41
+ end
42
+ end
43
+ end
44
+ end
@@ -1,5 +1,7 @@
1
1
  require 'autoscaler/sidekiq/queue_system'
2
2
  require 'autoscaler/sidekiq/celluloid_monitor'
3
+ require 'autoscaler/binary_scaling_strategy'
4
+ require 'autoscaler/delayed_shutdown'
3
5
 
4
6
  module Autoscaler
5
7
  module Sidekiq
@@ -7,12 +9,14 @@ module Autoscaler
7
9
  # Starts the monitor and notifies it of job events that may occur while it's sleeping
8
10
  class MonitorMiddlewareAdapter
9
11
  # @param [scaler] scaler object that actually performs scaling operations (e.g. {HerokuScaler})
10
- # @param [Numeric] timeout number of seconds to wait before shutdown
12
+ # @param [Strategy,Numeric] timeout strategy object that determines target workers, or a timeout to be passed to {DelayedShutdown}+{BinaryScalingStrategy}
11
13
  # @param [Array[String]] specified_queues list of queues to monitor to determine if there is work left. Defaults to all sidekiq queues.
12
14
  def initialize(scaler, timeout, specified_queues = nil)
13
- system = QueueSystem.new(specified_queues)
14
15
  unless monitor
15
- CelluloidMonitor.supervise_as(:autoscaler_monitor, scaler, timeout, system)
16
+ CelluloidMonitor.supervise_as(:autoscaler_monitor,
17
+ scaler,
18
+ strategy(timeout),
19
+ QueueSystem.new(specified_queues))
16
20
  end
17
21
  end
18
22
 
@@ -28,6 +32,14 @@ module Autoscaler
28
32
  def monitor
29
33
  Celluloid::Actor[:autoscaler_monitor]
30
34
  end
35
+
36
+ def strategy(timeout)
37
+ if timeout.respond_to?(:call)
38
+ timeout
39
+ else
40
+ DelayedShutdown.new(BinaryScalingStrategy.new, timeout)
41
+ end
42
+ end
31
43
  end
32
44
  end
33
45
  end
@@ -1,72 +1,20 @@
1
1
  require 'sidekiq/api'
2
+ require 'autoscaler/sidekiq/specified_queue_system'
3
+ require 'autoscaler/sidekiq/entire_queue_system'
2
4
 
3
5
  module Autoscaler
4
6
  module Sidekiq
5
7
  # Interface to to interrogate the queuing system
6
- class QueueSystem
8
+ # convenience constructor for SpecifiedQueueSystem and EntireQueueSystem
9
+ module QueueSystem
7
10
  # @param [Array[String]] specified_queues list of queues to monitor to determine if there is work left. Defaults to all sidekiq queues.
8
- def initialize(specified_queues = nil)
9
- @specified_queues = specified_queues
10
- end
11
-
12
- # @return [boolean] whether there is queued work - does not include work currently in progress
13
- def pending_work?
14
- refresh_sidekiq_queues!
15
- queued_work? || scheduled_work? || retry_work?
16
- end
17
-
18
- # @return [boolean] whether there are workers actively running
19
- def working?
20
- workers > 0
21
- end
22
-
23
- # @return [Array[String]]
24
- def queue_names
25
- (@specified_queues || sidekiq_queues.keys)
26
- end
27
-
28
- private
29
- def queued_work?
30
- queue_names.any? {|name| sidekiq_queues[name].to_i > 0 }
31
- end
32
-
33
- def scheduled_work?
34
- empty_sorted_set?("schedule")
35
- end
36
-
37
- def retry_work?
38
- empty_sorted_set?("retry")
39
- end
40
-
41
- attr_reader :sidekiq_queues
42
-
43
- def refresh_sidekiq_queues!
44
- @sidekiq_queues = ::Sidekiq::Stats.new.queues
45
- end
46
-
47
- def empty_sorted_set?(sorted_set)
48
- ss = ::Sidekiq::SortedSet.new(sorted_set)
49
- ss.any? { |job| queue_names.include?(job.queue) }
50
- end
51
-
52
- def workers
53
- if @specified_queues
54
- specified_workers
11
+ def self.new(specified_queues = nil)
12
+ if specified_queues
13
+ SpecifiedQueueSystem.new(specified_queues)
55
14
  else
56
- total_workers
15
+ EntireQueueSystem.new
57
16
  end
58
17
  end
59
-
60
- def total_workers
61
- ::Sidekiq::Workers.new.size
62
- end
63
-
64
- def specified_workers
65
- refresh_sidekiq_queues!
66
- ::Sidekiq::Workers.new.count {|name, work|
67
- queue_names.include?(work['queue'])
68
- }
69
- end
70
18
  end
71
19
  end
72
20
  end
@@ -37,7 +37,7 @@ module Autoscaler
37
37
  attr_reader :activity
38
38
 
39
39
  def pending_work?
40
- system.pending_work?
40
+ system.queued > 0 || system.scheduled > 0 || system.retrying > 0
41
41
  end
42
42
 
43
43
  def working!(queue)
@@ -0,0 +1,49 @@
1
+ require 'sidekiq/api'
2
+
3
+ module Autoscaler
4
+ module Sidekiq
5
+ # Interface to to interrogate the queuing system
6
+ # Includes only the queues provided to the constructor
7
+ class SpecifiedQueueSystem
8
+ # @param [Array[String]] specified_queues list of queues to monitor to determine if there is work left. Defaults to all sidekiq queues.
9
+ def initialize(specified_queues)
10
+ @queue_names = specified_queues
11
+ end
12
+
13
+ # @return [Integer] number of worker actively engaged
14
+ def workers
15
+ ::Sidekiq::Workers.new.count {|name, work|
16
+ queue_names.include?(work['queue'])
17
+ }
18
+ end
19
+
20
+ # @return [Integer] amount work ready to go
21
+ def queued
22
+ queue_names.map {|name| sidekiq_queues[name].to_i}.reduce(&:+)
23
+ end
24
+
25
+ # @return [Integer] amount of work scheduled for some time in the future
26
+ def scheduled
27
+ count_sorted_set("schedule")
28
+ end
29
+
30
+ # @return [Integer] amount of work still being retried
31
+ def retrying
32
+ count_sorted_set("retry")
33
+ end
34
+
35
+ # @return [Array[String]]
36
+ attr_reader :queue_names
37
+
38
+ private
39
+ def sidekiq_queues
40
+ ::Sidekiq::Stats.new.queues
41
+ end
42
+
43
+ def count_sorted_set(sorted_set)
44
+ ss = ::Sidekiq::SortedSet.new(sorted_set)
45
+ ss.count { |job| queue_names.include?(job.queue) }
46
+ end
47
+ end
48
+ end
49
+ end
@@ -1,4 +1,4 @@
1
1
  module Autoscaler
2
2
  # version number
3
- VERSION = "0.3.0"
3
+ VERSION = "0.4.0"
4
4
  end
@@ -0,0 +1,19 @@
1
+ require 'spec_helper'
2
+ require 'test_system'
3
+ require 'autoscaler/binary_scaling_strategy'
4
+
5
+ describe Autoscaler::BinaryScalingStrategy do
6
+ let(:cut) {Autoscaler::BinaryScalingStrategy}
7
+
8
+ it "scales with no work" do
9
+ system = TestSystem.new(0)
10
+ strategy = cut.new
11
+ strategy.call(system, 1).should == 0
12
+ end
13
+
14
+ it "does not scale with pending work" do
15
+ system = TestSystem.new(1)
16
+ strategy = cut.new(2)
17
+ strategy.call(system, 1).should == 2
18
+ end
19
+ end
@@ -1,16 +1,8 @@
1
1
  require 'spec_helper'
2
+ require 'test_system'
2
3
  require 'autoscaler/sidekiq/celluloid_monitor'
3
4
  require 'timeout'
4
5
 
5
- class TestSystem
6
- def initialize(pending)
7
- @pending = pending
8
- end
9
-
10
- def working?; false; end
11
- def pending_work?; @pending; end
12
- end
13
-
14
6
  describe Autoscaler::Sidekiq::CelluloidMonitor do
15
7
  before do
16
8
  @redis = Sidekiq.redis = REDIS
@@ -21,17 +13,17 @@ describe Autoscaler::Sidekiq::CelluloidMonitor do
21
13
  let(:scaler) {TestScaler.new(1)}
22
14
 
23
15
  it "scales with no work" do
24
- system = TestSystem.new(false)
25
- manager = cut.new(scaler, 0, system)
26
- Timeout.timeout(1) { manager.wait_for_downscale }
16
+ system = TestSystem.new(0)
17
+ manager = cut.new(scaler, lambda{|s,t| 0}, system)
18
+ Timeout.timeout(1) { manager.wait_for_downscale(0.5) }
27
19
  scaler.workers.should == 0
28
20
  manager.terminate
29
21
  end
30
22
 
31
23
  it "does not scale with pending work" do
32
- system = TestSystem.new(true)
33
- manager = cut.new(scaler, 0, system)
34
- expect {Timeout.timeout(1) { manager.wait_for_downscale }}.to raise_error Timeout::Error
24
+ system = TestSystem.new(1)
25
+ manager = cut.new(scaler, lambda{|s,t| 1}, system)
26
+ expect {Timeout.timeout(1) { manager.wait_for_downscale(0.5) }}.to raise_error Timeout::Error
35
27
  scaler.workers.should == 1
36
28
  manager.terminate
37
29
  end
@@ -0,0 +1,61 @@
1
+ require 'spec_helper'
2
+ require 'autoscaler/sidekiq/entire_queue_system'
3
+
4
+ describe Autoscaler::Sidekiq::EntireQueueSystem do
5
+ let(:cut) {Autoscaler::Sidekiq::EntireQueueSystem}
6
+
7
+ before do
8
+ @redis = Sidekiq.redis = REDIS
9
+ Sidekiq.redis {|c| c.flushdb }
10
+ end
11
+
12
+ def with_work_in_set(queue, set)
13
+ payload = Sidekiq.dump_json('queue' => queue)
14
+ Sidekiq.redis { |c| c.zadd(set, (Time.now.to_f + 30.to_f).to_s, payload)}
15
+ end
16
+
17
+ def with_scheduled_work_in(queue)
18
+ with_work_in_set(queue, 'schedule')
19
+ end
20
+
21
+ def with_retry_work_in(queue)
22
+ with_work_in_set(queue, 'retry')
23
+ end
24
+
25
+ subject {cut.new}
26
+
27
+ it {subject.queue_names.should == []}
28
+ it {subject.workers.should == 0}
29
+
30
+ describe 'no queued work' do
31
+ it "with no work" do
32
+ subject.stub(:sidekiq_queues).and_return({'queue' => 0, 'another_queue' => 0})
33
+ subject.queued.should == 0
34
+ end
35
+
36
+ it "with no scheduled work" do
37
+ subject.scheduled.should == 0
38
+ end
39
+
40
+ it "with no retry work" do
41
+ subject.retrying.should == 0
42
+ end
43
+ end
44
+
45
+ describe 'with queued work' do
46
+ it "with enqueued work" do
47
+ subject.stub(:sidekiq_queues).and_return({'queue' => 1})
48
+ subject.queued.should == 1
49
+ end
50
+
51
+ it "with schedule work" do
52
+ with_scheduled_work_in('queue')
53
+ subject.scheduled.should == 1
54
+ end
55
+
56
+ it "with retry work" do
57
+ with_retry_work_in('queue')
58
+ subject.retrying.should == 1
59
+ end
60
+ end
61
+ end
@@ -1,8 +1,8 @@
1
1
  require 'spec_helper'
2
- require 'autoscaler/sidekiq/queue_system'
2
+ require 'autoscaler/sidekiq/specified_queue_system'
3
3
 
4
- describe Autoscaler::Sidekiq::QueueSystem do
5
- let(:cut) {Autoscaler::Sidekiq::QueueSystem}
4
+ describe Autoscaler::Sidekiq::SpecifiedQueueSystem do
5
+ let(:cut) {Autoscaler::Sidekiq::SpecifiedQueueSystem}
6
6
 
7
7
  before do
8
8
  @redis = Sidekiq.redis = REDIS
@@ -25,39 +25,39 @@ describe Autoscaler::Sidekiq::QueueSystem do
25
25
  subject {cut.new(['queue'])}
26
26
 
27
27
  it {subject.queue_names.should == ['queue']}
28
- it {subject.working?.should be_false}
28
+ it {subject.workers.should == 0}
29
29
 
30
- describe 'no pending work' do
30
+ describe 'no queued work' do
31
31
  it "with no work" do
32
32
  subject.stub(:sidekiq_queues).and_return({'queue' => 0, 'another_queue' => 1})
33
- subject.should_not be_pending_work
33
+ subject.queued.should == 0
34
34
  end
35
35
 
36
36
  it "with scheduled work in another queue" do
37
37
  with_scheduled_work_in('another_queue')
38
- subject.should_not be_pending_work
38
+ subject.scheduled.should == 0
39
39
  end
40
40
 
41
41
  it "with retry work in another queue" do
42
42
  with_retry_work_in('another_queue')
43
- subject.should_not be_pending_work
43
+ subject.retrying.should == 0
44
44
  end
45
45
  end
46
46
 
47
- describe 'with pending work' do
47
+ describe 'with queued work' do
48
48
  it "with enqueued work" do
49
49
  subject.stub(:sidekiq_queues).and_return({'queue' => 1})
50
- subject.should be_pending_work
50
+ subject.queued.should == 1
51
51
  end
52
52
 
53
53
  it "with schedule work" do
54
54
  with_scheduled_work_in('queue')
55
- subject.should be_pending_work
55
+ subject.scheduled.should == 1
56
56
  end
57
57
 
58
58
  it "with retry work" do
59
59
  with_retry_work_in('queue')
60
- subject.should be_pending_work
60
+ subject.retrying.should == 1
61
61
  end
62
62
  end
63
63
  end
@@ -0,0 +1,10 @@
1
+ class TestSystem
2
+ def initialize(pending)
3
+ @pending = pending
4
+ end
5
+
6
+ def workers; 0; end
7
+ def queued; @pending; end
8
+ def scheduled; 0; end
9
+ def retrying; 0; end
10
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: autoscaler
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.0
4
+ version: 0.4.0
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2013-05-18 00:00:00.000000000 Z
13
+ date: 2013-06-21 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: sidekiq
@@ -132,13 +132,16 @@ executables: []
132
132
  extensions: []
133
133
  extra_rdoc_files: []
134
134
  files:
135
+ - lib/autoscaler/binary_scaling_strategy.rb
135
136
  - lib/autoscaler/heroku_scaler.rb
136
137
  - lib/autoscaler/sidekiq/activity.rb
137
138
  - lib/autoscaler/sidekiq/celluloid_monitor.rb
138
139
  - lib/autoscaler/sidekiq/client.rb
140
+ - lib/autoscaler/sidekiq/entire_queue_system.rb
139
141
  - lib/autoscaler/sidekiq/monitor_middleware_adapter.rb
140
142
  - lib/autoscaler/sidekiq/queue_system.rb
141
143
  - lib/autoscaler/sidekiq/sleep_wait_server.rb
144
+ - lib/autoscaler/sidekiq/specified_queue_system.rb
142
145
  - lib/autoscaler/sidekiq.rb
143
146
  - lib/autoscaler/stub_scaler.rb
144
147
  - lib/autoscaler/version.rb
@@ -148,15 +151,18 @@ files:
148
151
  - examples/complex.rb
149
152
  - examples/simple.rb
150
153
  - Guardfile
154
+ - spec/autoscaler/binary_scaling_strategy_spec.rb
151
155
  - spec/autoscaler/heroku_scaler_spec.rb
152
156
  - spec/autoscaler/sidekiq/activity_spec.rb
153
157
  - spec/autoscaler/sidekiq/celluloid_monitor_spec.rb
154
158
  - spec/autoscaler/sidekiq/client_spec.rb
159
+ - spec/autoscaler/sidekiq/entire_queue_system_spec.rb
155
160
  - spec/autoscaler/sidekiq/monitor_middleware_adapter_spec.rb
156
- - spec/autoscaler/sidekiq/queue_system_spec.rb
157
161
  - spec/autoscaler/sidekiq/sleep_wait_server_spec.rb
162
+ - spec/autoscaler/sidekiq/specified_queue_system_spec.rb
158
163
  - spec/redis_test.conf
159
164
  - spec/spec_helper.rb
165
+ - spec/test_system.rb
160
166
  homepage: ''
161
167
  licenses: []
162
168
  post_install_message:
@@ -183,13 +189,16 @@ specification_version: 3
183
189
  summary: Start/stop Sidekiq workers on Heroku
184
190
  test_files:
185
191
  - Guardfile
192
+ - spec/autoscaler/binary_scaling_strategy_spec.rb
186
193
  - spec/autoscaler/heroku_scaler_spec.rb
187
194
  - spec/autoscaler/sidekiq/activity_spec.rb
188
195
  - spec/autoscaler/sidekiq/celluloid_monitor_spec.rb
189
196
  - spec/autoscaler/sidekiq/client_spec.rb
197
+ - spec/autoscaler/sidekiq/entire_queue_system_spec.rb
190
198
  - spec/autoscaler/sidekiq/monitor_middleware_adapter_spec.rb
191
- - spec/autoscaler/sidekiq/queue_system_spec.rb
192
199
  - spec/autoscaler/sidekiq/sleep_wait_server_spec.rb
200
+ - spec/autoscaler/sidekiq/specified_queue_system_spec.rb
193
201
  - spec/redis_test.conf
194
202
  - spec/spec_helper.rb
203
+ - spec/test_system.rb
195
204
  has_rdoc: