autoscaler 0.2.1 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,9 +1,15 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.3.0
4
+
5
+ - Downscale method changed from busy-waiting workers to a separate monitor process
6
+ - Minimum Sidekiq version raised to 2.7 to take advantage of Worker API
7
+ - Internal refactoring
8
+ - Autoscaler::StubScaler may be used for local testing
9
+
3
10
  ## 0.2.1
4
11
 
5
12
  - Separate background activity flags to avoid crosstalk between processes
6
- - Autoscaler::StubScaler may be used for local testing
7
13
 
8
14
  ## 0.2.0
9
15
 
data/README.md CHANGED
@@ -19,6 +19,9 @@ This gem uses the [Herkou-Api](https://github.com/heroku/heroku.rb) gem, which r
19
19
 
20
20
  Install the middleware in your `Sidekiq.configure_` blocks
21
21
 
22
+ require 'autoscaler/sidekiq'
23
+ require 'autoscaler/heroku_scaler'
24
+
22
25
  Sidekiq.configure_client do |config|
23
26
  config.client_middleware do |chain|
24
27
  chain.add Autoscaler::Sidekiq::Client, 'default' => Autoscaler::HerokuScaler.new
@@ -33,17 +36,12 @@ Install the middleware in your `Sidekiq.configure_` blocks
33
36
 
34
37
  ## Limits and Challenges
35
38
 
36
- - HerokuScaler includes an attempt at current-worker cache that may be overcomplication, and doesn't work very well (see next)
37
- - Multiple threads often send scaling requests at once. Heroku seems to handle this well.
38
- - Workers sleep-loop and are not actually returned to the pool; when a job or timeout happen, they can all release at once.
39
- - If you set job-timeouts on your tasks, they will likely trigger on the sleep-loop (see previous).
39
+ - HerokuScaler includes an attempt at current-worker cache that may be overcomplication, and doesn't work very well on the server
40
+ - Multiple scale-down loops may be started, particularly if there are multiple jobs queued when the servers comes up. Heroku seems to handle multiple scale-down commands well.
41
+ - The scale-down monitor is triggered on job completion (and server middleware is only run around jobs), so if the server nevers processes any jobs, it won't turn off.
40
42
  - The retry and schedule lists are considered - if you schedule a long-running task, the process will not scale-down.
41
43
  - If background jobs trigger jobs in other scaled processes, please note you'll need `config.client_middleware` in your `Sidekiq.configure_server` block in order to scale-up.
42
44
 
43
- ### Long Jobs
44
-
45
- Since the shutdown check gets performed every time a job completes, the shutdown-timeout will need to be longer than the longest job. For mixed workloads, you might want to have multiple sidekiq processes defined. I use one with many workers for general work, and a single-worker process for long import jobs. See `examples/complex.rb`
46
-
47
45
  ## Tests
48
46
 
49
47
  The project is setup to run RSpec with Guard. It expects a redis instance on a custom port, which is started by the Guardfile.
@@ -1,114 +1,11 @@
1
- require 'sidekiq'
1
+ require 'autoscaler/sidekiq/client'
2
+ require 'autoscaler/sidekiq/monitor_middleware_adapter'
2
3
 
3
4
  module Autoscaler
4
5
  # namespace module for Sidekiq middlewares
5
6
  module Sidekiq
6
- # Sidekiq client middleware
7
- # Performs scale-up when items are queued and there are no workers running
8
- class Client
9
- # @param [Hash] scalers map of queue(String) => scaler (e.g. {HerokuScaler}).
10
- # Which scaler to use for each sidekiq queue
11
- def initialize(scalers)
12
- @scalers = scalers
13
- end
14
-
15
- # Sidekiq middleware api method
16
- def call(worker_class, item, queue)
17
- @scalers[queue] && @scalers[queue].workers = 1
18
- yield
19
- end
20
- end
21
-
22
7
  # Sidekiq server middleware
23
8
  # Performs scale-down when the queue is empty
24
- class Server
25
- # @param [scaler] scaler object that actually performs scaling operations (e.g. {HerokuScaler})
26
- # @param [Numeric] timeout number of seconds to wait before shutdown
27
- # @param [Array[String]] specified_queues list of queues to monitor to determine if there is work left. Defaults to all sidekiq queues.
28
- def initialize(scaler, timeout, specified_queues = nil)
29
- @scaler = scaler
30
- @timeout = timeout
31
- @specified_queues = specified_queues
32
- end
33
-
34
- # Sidekiq middleware api entry point
35
- def call(worker, msg, queue)
36
- working!(queue)
37
- yield
38
- ensure
39
- working!(queue)
40
- wait_for_task_or_scale
41
- end
42
-
43
- private
44
- def refresh_sidekiq_queues!
45
- @sidekiq_queues = ::Sidekiq::Stats.new.queues
46
- end
47
-
48
- attr_reader :sidekiq_queues
49
-
50
- def queue_names
51
- (@specified_queues || sidekiq_queues.keys)
52
- end
53
-
54
- def queued_work?
55
- queue_names.any? {|name| sidekiq_queues[name].to_i > 0 }
56
- end
57
-
58
- def scheduled_work?
59
- empty_sorted_set?("schedule")
60
- end
61
-
62
- def retry_work?
63
- empty_sorted_set?("retry")
64
- end
65
-
66
- def empty_sorted_set?(sorted_set)
67
- ss = ::Sidekiq::SortedSet.new(sorted_set)
68
- ss.any? { |job| queue_names.include?(job.queue) }
69
- end
70
-
71
- def pending_work?
72
- refresh_sidekiq_queues!
73
- queued_work? || scheduled_work? || retry_work?
74
- end
75
-
76
- def wait_for_task_or_scale
77
- loop do
78
- return if pending_work?
79
- return @scaler.workers = 0 if idle?
80
- sleep(0.5)
81
- end
82
- end
83
-
84
- def working!(queue)
85
- active_at queue, Time.now
86
- end
87
-
88
- # test support
89
- def idle!(queue)
90
- active_at queue, Time.now - @timeout*2
91
- end
92
-
93
- def idle_time
94
- t = last_activity
95
- return 0 unless t
96
- Time.now - Time.parse(t)
97
- end
98
-
99
- def idle?
100
- idle_time > @timeout
101
- end
102
-
103
- def last_activity
104
- ::Sidekiq.redis {|c|
105
- queue_names.map {|q| c.get('background_activity:'+q)}.compact.max
106
- }
107
- end
108
-
109
- def active_at(queue, time)
110
- ::Sidekiq.redis {|c| c.set('background_activity:'+queue, time)}
111
- end
112
- end
9
+ Server = MonitorMiddlewareAdapter
113
10
  end
114
11
  end
@@ -0,0 +1,51 @@
1
+ require 'sidekiq'
2
+
3
+ module Autoscaler
4
+ module Sidekiq
5
+ # Tracks activity timeouts using Sidekiq's redis connection
6
+ class Activity
7
+ # @param [Numeric] timeout number of seconds to wait before shutdown
8
+ def initialize(timeout)
9
+ @timeout = timeout
10
+ end
11
+
12
+ # Record that a queue has activity
13
+ # @param [String] queue
14
+ def working!(queue)
15
+ active_at queue, Time.now
16
+ end
17
+
18
+ # Record that a queue is idle and timed out - mostly for test support
19
+ # @param [String] queue
20
+ def idle!(queue)
21
+ active_at queue, Time.now - timeout*2
22
+ end
23
+
24
+ # Have the watched queues timed out?
25
+ # @param [Array[String]] queues list of queues to monitor to determine if there is work left
26
+ # @return [boolean]
27
+ def idle?(queues)
28
+ idle_time(queues) > timeout
29
+ end
30
+
31
+ private
32
+ attr_reader :timeout
33
+
34
+ def idle_time(queues)
35
+ t = last_activity(queues)
36
+ return 0 unless t
37
+ Time.now - Time.parse(t)
38
+ end
39
+
40
+ def last_activity(queues)
41
+ ::Sidekiq.redis {|c|
42
+ queues.map {|q| c.get('background_activity:'+q)}.compact.max
43
+ }
44
+ end
45
+
46
+ def active_at(queue, time)
47
+ ::Sidekiq.redis {|c| c.set('background_activity:'+queue, time)}
48
+ end
49
+ end
50
+ end
51
+ end
@@ -0,0 +1,75 @@
1
+ require 'celluloid'
2
+
3
+ module Autoscaler
4
+ module Sidekiq
5
+ # Actor to monitor the sidekiq server for scale-down
6
+ class CelluloidMonitor
7
+ include Celluloid
8
+
9
+ # @param [scaler] scaler object that actually performs scaling operations (e.g. {HerokuScaler})
10
+ # @param [Numeric] timeout number of seconds to wait before shutdown
11
+ # @param [System] system interface to the queuing system that provides `pending_work?`
12
+ def initialize(scaler, timeout, system)
13
+ @scaler = scaler
14
+ @timeout = timeout
15
+ @poll = [timeout/4.0, 0.5].max
16
+ @system = system
17
+ @running = false
18
+ end
19
+
20
+ # Mostly sleep until there has been no activity for the timeout
21
+ def wait_for_downscale
22
+ once do
23
+ active_now!
24
+
25
+ while active? || time_left?
26
+ sleep(@poll)
27
+ update_activity
28
+ end
29
+
30
+ @scaler.workers = 0
31
+ end
32
+ end
33
+
34
+ # Notify the monitor that a job is starting
35
+ def starting_job
36
+ end
37
+
38
+ # Notify the monitor that a job has finished
39
+ def finished_job
40
+ active_now!
41
+ async.wait_for_downscale
42
+ end
43
+
44
+ private
45
+ attr_reader :system
46
+
47
+ def active?
48
+ system.pending_work? || system.working?
49
+ end
50
+
51
+ def update_activity
52
+ active_now! if active?
53
+ end
54
+
55
+ def active_now!
56
+ @activity = Time.now
57
+ end
58
+
59
+ def time_left?
60
+ (Time.now - @activity) < @timeout
61
+ end
62
+
63
+ def once
64
+ return if @running
65
+
66
+ begin
67
+ @running = true
68
+ yield
69
+ ensure
70
+ @running = false
71
+ end
72
+ end
73
+ end
74
+ end
75
+ end
@@ -0,0 +1,19 @@
1
+ module Autoscaler
2
+ module Sidekiq
3
+ # Sidekiq client middleware
4
+ # Performs scale-up when items are queued and there are no workers running
5
+ class Client
6
+ # @param [Hash] scalers map of queue(String) => scaler (e.g. {HerokuScaler}).
7
+ # Which scaler to use for each sidekiq queue
8
+ def initialize(scalers)
9
+ @scalers = scalers
10
+ end
11
+
12
+ # Sidekiq middleware api method
13
+ def call(worker_class, item, queue)
14
+ @scalers[queue] && @scalers[queue].workers = 1
15
+ yield
16
+ end
17
+ end
18
+ end
19
+ end
@@ -0,0 +1,33 @@
1
+ require 'autoscaler/sidekiq/queue_system'
2
+ require 'autoscaler/sidekiq/celluloid_monitor'
3
+
4
+ module Autoscaler
5
+ module Sidekiq
6
+ # Shim to the existing autoscaler interface
7
+ # Starts the monitor and notifies it of job events that may occur while it's sleeping
8
+ class MonitorMiddlewareAdapter
9
+ # @param [scaler] scaler object that actually performs scaling operations (e.g. {HerokuScaler})
10
+ # @param [Numeric] timeout number of seconds to wait before shutdown
11
+ # @param [Array[String]] specified_queues list of queues to monitor to determine if there is work left. Defaults to all sidekiq queues.
12
+ def initialize(scaler, timeout, specified_queues = nil)
13
+ system = QueueSystem.new(specified_queues)
14
+ unless monitor
15
+ CelluloidMonitor.supervise_as(:autoscaler_monitor, scaler, timeout, system)
16
+ end
17
+ end
18
+
19
+ # Sidekiq middleware api entry point
20
+ def call(worker, msg, queue)
21
+ monitor.async.starting_job
22
+ yield
23
+ ensure
24
+ monitor.async.finished_job
25
+ end
26
+
27
+ private
28
+ def monitor
29
+ Celluloid::Actor[:autoscaler_monitor]
30
+ end
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,72 @@
1
+ require 'sidekiq/api'
2
+
3
+ module Autoscaler
4
+ module Sidekiq
5
+ # Interface to to interrogate the queuing system
6
+ class QueueSystem
7
+ # @param [Array[String]] specified_queues list of queues to monitor to determine if there is work left. Defaults to all sidekiq queues.
8
+ def initialize(specified_queues = nil)
9
+ @specified_queues = specified_queues
10
+ end
11
+
12
+ # @return [boolean] whether there is queued work - does not include work currently in progress
13
+ def pending_work?
14
+ refresh_sidekiq_queues!
15
+ queued_work? || scheduled_work? || retry_work?
16
+ end
17
+
18
+ # @return [boolean] whether there are workers actively running
19
+ def working?
20
+ workers > 0
21
+ end
22
+
23
+ # @return [Array[String]]
24
+ def queue_names
25
+ (@specified_queues || sidekiq_queues.keys)
26
+ end
27
+
28
+ private
29
+ def queued_work?
30
+ queue_names.any? {|name| sidekiq_queues[name].to_i > 0 }
31
+ end
32
+
33
+ def scheduled_work?
34
+ empty_sorted_set?("schedule")
35
+ end
36
+
37
+ def retry_work?
38
+ empty_sorted_set?("retry")
39
+ end
40
+
41
+ attr_reader :sidekiq_queues
42
+
43
+ def refresh_sidekiq_queues!
44
+ @sidekiq_queues = ::Sidekiq::Stats.new.queues
45
+ end
46
+
47
+ def empty_sorted_set?(sorted_set)
48
+ ss = ::Sidekiq::SortedSet.new(sorted_set)
49
+ ss.any? { |job| queue_names.include?(job.queue) }
50
+ end
51
+
52
+ def workers
53
+ if @specified_queues
54
+ specified_workers
55
+ else
56
+ total_workers
57
+ end
58
+ end
59
+
60
+ def total_workers
61
+ ::Sidekiq::Workers.new.size
62
+ end
63
+
64
+ def specified_workers
65
+ refresh_sidekiq_queues!
66
+ ::Sidekiq::Workers.new.count {|name, work|
67
+ queue_names.include?(work['queue'])
68
+ }
69
+ end
70
+ end
71
+ end
72
+ end
@@ -0,0 +1,52 @@
1
+ require 'autoscaler/sidekiq/queue_system'
2
+ require 'autoscaler/sidekiq/activity'
3
+
4
+ module Autoscaler
5
+ module Sidekiq
6
+ # Sidekiq server middleware
7
+ # Performs scale-down when the queue is empty
8
+ class SleepWaitServer
9
+ # @param [scaler] scaler object that actually performs scaling operations (e.g. {HerokuScaler})
10
+ # @param [Numeric] timeout number of seconds to wait before shutdown
11
+ # @param [Array[String]] specified_queues list of queues to monitor to determine if there is work left. Defaults to all sidekiq queues.
12
+ def initialize(scaler, timeout, specified_queues = nil)
13
+ @scaler = scaler
14
+ @activity = Activity.new(timeout)
15
+ @system = QueueSystem.new(specified_queues)
16
+ end
17
+
18
+ # Sidekiq middleware api entry point
19
+ def call(worker, msg, queue)
20
+ working!(queue)
21
+ yield
22
+ ensure
23
+ working!(queue)
24
+ wait_for_task_or_scale
25
+ end
26
+
27
+ private
28
+ def wait_for_task_or_scale
29
+ loop do
30
+ return if pending_work?
31
+ return @scaler.workers = 0 if idle?
32
+ sleep(0.5)
33
+ end
34
+ end
35
+
36
+ attr_reader :system
37
+ attr_reader :activity
38
+
39
+ def pending_work?
40
+ system.pending_work?
41
+ end
42
+
43
+ def working!(queue)
44
+ activity.working!(queue)
45
+ end
46
+
47
+ def idle?
48
+ activity.idle?(system.queue_names)
49
+ end
50
+ end
51
+ end
52
+ end
@@ -0,0 +1,25 @@
1
+ require 'heroku-api'
2
+
3
+ module Autoscaler
4
+ # A minimal scaler to use as stub for local testing
5
+ class StubScaler
6
+ # @param [String] type used to distinguish messages from multiple stubs
7
+ def initialize(type = 'worker')
8
+ @type = type
9
+ @workers = 0
10
+ end
11
+
12
+ attr_reader :type
13
+
14
+ # Read the current worker count
15
+ # @return [Numeric] number of workers
16
+ attr_reader :workers
17
+
18
+ # Set the number of workers
19
+ # @param [Numeric] n number of workers
20
+ def workers=(n)
21
+ p "Scaling #{type} to #{n}"
22
+ @workers = n
23
+ end
24
+ end
25
+ end
@@ -1,4 +1,4 @@
1
1
  module Autoscaler
2
2
  # version number
3
- VERSION = "0.2.1"
3
+ VERSION = "0.3.0"
4
4
  end
@@ -0,0 +1,21 @@
1
+ require 'spec_helper'
2
+ require 'autoscaler/sidekiq/activity'
3
+
4
+ describe Autoscaler::Sidekiq::Activity do
5
+ before do
6
+ @redis = Sidekiq.redis = REDIS
7
+ Sidekiq.redis {|c| c.flushdb }
8
+ end
9
+
10
+ let(:cut) {Autoscaler::Sidekiq::Activity}
11
+ let(:activity) {cut.new(0)}
12
+
13
+ context "when another process is working" do
14
+ let(:other_process) {cut.new(10)}
15
+ before do
16
+ activity.idle!('queue')
17
+ other_process.working!('other_queue')
18
+ end
19
+ it {activity.should be_idle(['queue'])}
20
+ end
21
+ end
@@ -0,0 +1,38 @@
1
+ require 'spec_helper'
2
+ require 'autoscaler/sidekiq/celluloid_monitor'
3
+ require 'timeout'
4
+
5
+ class TestSystem
6
+ def initialize(pending)
7
+ @pending = pending
8
+ end
9
+
10
+ def working?; false; end
11
+ def pending_work?; @pending; end
12
+ end
13
+
14
+ describe Autoscaler::Sidekiq::CelluloidMonitor do
15
+ before do
16
+ @redis = Sidekiq.redis = REDIS
17
+ Sidekiq.redis {|c| c.flushdb }
18
+ end
19
+
20
+ let(:cut) {Autoscaler::Sidekiq::CelluloidMonitor}
21
+ let(:scaler) {TestScaler.new(1)}
22
+
23
+ it "scales with no work" do
24
+ system = TestSystem.new(false)
25
+ manager = cut.new(scaler, 0, system)
26
+ Timeout.timeout(1) { manager.wait_for_downscale }
27
+ scaler.workers.should == 0
28
+ manager.terminate
29
+ end
30
+
31
+ it "does not scale with pending work" do
32
+ system = TestSystem.new(true)
33
+ manager = cut.new(scaler, 0, system)
34
+ expect {Timeout.timeout(1) { manager.wait_for_downscale }}.to raise_error Timeout::Error
35
+ scaler.workers.should == 1
36
+ manager.terminate
37
+ end
38
+ end
@@ -0,0 +1,15 @@
1
+ require 'spec_helper'
2
+ require 'autoscaler/sidekiq/client'
3
+
4
+ describe Autoscaler::Sidekiq::Client do
5
+ let(:cut) {Autoscaler::Sidekiq::Client}
6
+ let(:scaler) {TestScaler.new(0)}
7
+ let(:client) {cut.new('queue' => scaler)}
8
+
9
+ it 'scales' do
10
+ client.call(Class, {}, 'queue') {}
11
+ scaler.workers.should == 1
12
+ end
13
+
14
+ it('yields') {client.call(Class, {}, 'queue') {:foo}.should == :foo}
15
+ end
@@ -0,0 +1,15 @@
1
+ require 'spec_helper'
2
+ require 'autoscaler/sidekiq/monitor_middleware_adapter'
3
+
4
+ describe Autoscaler::Sidekiq::MonitorMiddlewareAdapter do
5
+ before do
6
+ @redis = Sidekiq.redis = REDIS
7
+ Sidekiq.redis {|c| c.flushdb }
8
+ end
9
+
10
+ let(:cut) {Autoscaler::Sidekiq::MonitorMiddlewareAdapter}
11
+ let(:scaler) {TestScaler.new(1)}
12
+ let(:server) {cut.new(scaler, 0, ['queue'])}
13
+
14
+ it('yields') {server.call(Object.new, {}, 'queue') {:foo}.should == :foo}
15
+ end
@@ -0,0 +1,63 @@
1
+ require 'spec_helper'
2
+ require 'autoscaler/sidekiq/queue_system'
3
+
4
+ describe Autoscaler::Sidekiq::QueueSystem do
5
+ let(:cut) {Autoscaler::Sidekiq::QueueSystem}
6
+
7
+ before do
8
+ @redis = Sidekiq.redis = REDIS
9
+ Sidekiq.redis {|c| c.flushdb }
10
+ end
11
+
12
+ def with_work_in_set(queue, set)
13
+ payload = Sidekiq.dump_json('queue' => queue)
14
+ Sidekiq.redis { |c| c.zadd(set, (Time.now.to_f + 30.to_f).to_s, payload)}
15
+ end
16
+
17
+ def with_scheduled_work_in(queue)
18
+ with_work_in_set(queue, 'schedule')
19
+ end
20
+
21
+ def with_retry_work_in(queue)
22
+ with_work_in_set(queue, 'retry')
23
+ end
24
+
25
+ subject {cut.new(['queue'])}
26
+
27
+ it {subject.queue_names.should == ['queue']}
28
+ it {subject.working?.should be_false}
29
+
30
+ describe 'no pending work' do
31
+ it "with no work" do
32
+ subject.stub(:sidekiq_queues).and_return({'queue' => 0, 'another_queue' => 1})
33
+ subject.should_not be_pending_work
34
+ end
35
+
36
+ it "with scheduled work in another queue" do
37
+ with_scheduled_work_in('another_queue')
38
+ subject.should_not be_pending_work
39
+ end
40
+
41
+ it "with retry work in another queue" do
42
+ with_retry_work_in('another_queue')
43
+ subject.should_not be_pending_work
44
+ end
45
+ end
46
+
47
+ describe 'with pending work' do
48
+ it "with enqueued work" do
49
+ subject.stub(:sidekiq_queues).and_return({'queue' => 1})
50
+ subject.should be_pending_work
51
+ end
52
+
53
+ it "with schedule work" do
54
+ with_scheduled_work_in('queue')
55
+ subject.should be_pending_work
56
+ end
57
+
58
+ it "with retry work" do
59
+ with_retry_work_in('queue')
60
+ subject.should be_pending_work
61
+ end
62
+ end
63
+ end
@@ -0,0 +1,31 @@
1
+ require 'spec_helper'
2
+ require 'autoscaler/sidekiq/sleep_wait_server'
3
+
4
+ describe Autoscaler::Sidekiq::SleepWaitServer do
5
+ before do
6
+ @redis = Sidekiq.redis = REDIS
7
+ Sidekiq.redis {|c| c.flushdb }
8
+ end
9
+
10
+ let(:cut) {Autoscaler::Sidekiq::SleepWaitServer}
11
+ let(:scaler) {TestScaler.new(1)}
12
+ let(:server) {cut.new(scaler, 0, ['queue'])}
13
+
14
+ def when_run
15
+ server.call(Object.new, {}, 'queue') {}
16
+ end
17
+
18
+ it "scales with no work" do
19
+ server.stub(:pending_work?).and_return(false)
20
+ when_run
21
+ scaler.workers.should == 0
22
+ end
23
+
24
+ it "does not scale with pending work" do
25
+ server.stub(:pending_work?).and_return(true)
26
+ when_run
27
+ scaler.workers.should == 1
28
+ end
29
+
30
+ it('yields') {server.call(Object.new, {}, 'queue') {:foo}.should == :foo}
31
+ end
@@ -6,3 +6,11 @@ RSpec.configure do |config|
6
6
 
7
7
  config.filter_run_excluding :online => true unless ENV['HEROKU_APP']
8
8
  end
9
+
10
+ class TestScaler
11
+ attr_accessor :workers
12
+
13
+ def initialize(n = 0)
14
+ self.workers = n
15
+ end
16
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: autoscaler
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.1
4
+ version: 0.3.0
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -10,30 +10,24 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2013-03-14 00:00:00.000000000 Z
13
+ date: 2013-05-18 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: sidekiq
17
17
  requirement: !ruby/object:Gem::Requirement
18
18
  none: false
19
19
  requirements:
20
- - - ! '>='
21
- - !ruby/object:Gem::Version
22
- version: 2.6.1
23
- - - <
20
+ - - ~>
24
21
  - !ruby/object:Gem::Version
25
- version: '3.0'
22
+ version: '2.7'
26
23
  type: :runtime
27
24
  prerelease: false
28
25
  version_requirements: !ruby/object:Gem::Requirement
29
26
  none: false
30
27
  requirements:
31
- - - ! '>='
32
- - !ruby/object:Gem::Version
33
- version: 2.6.1
34
- - - <
28
+ - - ~>
35
29
  - !ruby/object:Gem::Version
36
- version: '3.0'
30
+ version: '2.7'
37
31
  - !ruby/object:Gem::Dependency
38
32
  name: heroku-api
39
33
  requirement: !ruby/object:Gem::Requirement
@@ -139,7 +133,14 @@ extensions: []
139
133
  extra_rdoc_files: []
140
134
  files:
141
135
  - lib/autoscaler/heroku_scaler.rb
136
+ - lib/autoscaler/sidekiq/activity.rb
137
+ - lib/autoscaler/sidekiq/celluloid_monitor.rb
138
+ - lib/autoscaler/sidekiq/client.rb
139
+ - lib/autoscaler/sidekiq/monitor_middleware_adapter.rb
140
+ - lib/autoscaler/sidekiq/queue_system.rb
141
+ - lib/autoscaler/sidekiq/sleep_wait_server.rb
142
142
  - lib/autoscaler/sidekiq.rb
143
+ - lib/autoscaler/stub_scaler.rb
143
144
  - lib/autoscaler/version.rb
144
145
  - lib/autoscaler.rb
145
146
  - README.md
@@ -148,9 +149,14 @@ files:
148
149
  - examples/simple.rb
149
150
  - Guardfile
150
151
  - spec/autoscaler/heroku_scaler_spec.rb
151
- - spec/autoscaler/sidekiq_spec.rb
152
- - spec/spec_helper.rb
152
+ - spec/autoscaler/sidekiq/activity_spec.rb
153
+ - spec/autoscaler/sidekiq/celluloid_monitor_spec.rb
154
+ - spec/autoscaler/sidekiq/client_spec.rb
155
+ - spec/autoscaler/sidekiq/monitor_middleware_adapter_spec.rb
156
+ - spec/autoscaler/sidekiq/queue_system_spec.rb
157
+ - spec/autoscaler/sidekiq/sleep_wait_server_spec.rb
153
158
  - spec/redis_test.conf
159
+ - spec/spec_helper.rb
154
160
  homepage: ''
155
161
  licenses: []
156
162
  post_install_message:
@@ -178,7 +184,12 @@ summary: Start/stop Sidekiq workers on Heroku
178
184
  test_files:
179
185
  - Guardfile
180
186
  - spec/autoscaler/heroku_scaler_spec.rb
181
- - spec/autoscaler/sidekiq_spec.rb
182
- - spec/spec_helper.rb
187
+ - spec/autoscaler/sidekiq/activity_spec.rb
188
+ - spec/autoscaler/sidekiq/celluloid_monitor_spec.rb
189
+ - spec/autoscaler/sidekiq/client_spec.rb
190
+ - spec/autoscaler/sidekiq/monitor_middleware_adapter_spec.rb
191
+ - spec/autoscaler/sidekiq/queue_system_spec.rb
192
+ - spec/autoscaler/sidekiq/sleep_wait_server_spec.rb
183
193
  - spec/redis_test.conf
194
+ - spec/spec_helper.rb
184
195
  has_rdoc:
@@ -1,137 +0,0 @@
1
- require 'spec_helper'
2
- require 'autoscaler/sidekiq'
3
-
4
- class Scaler
5
- attr_accessor :workers
6
-
7
- def initialize(n = 0)
8
- self.workers = n
9
- end
10
- end
11
-
12
- class Autoscaler::Sidekiq::Server
13
- public :idle!
14
- public :working!
15
- end
16
-
17
- describe Autoscaler::Sidekiq do
18
- before do
19
- @redis = Sidekiq.redis = REDIS
20
- Sidekiq.redis {|c| c.flushdb }
21
- end
22
-
23
- let(:scaler) do
24
- Scaler.new(workers)
25
- end
26
-
27
- describe Autoscaler::Sidekiq::Client do
28
- let(:cut) {Autoscaler::Sidekiq::Client}
29
- let(:client) {cut.new('queue' => scaler)}
30
- let(:workers) {0}
31
-
32
- describe 'scales' do
33
- before {client.call(Class, {}, 'queue') {}}
34
- it {scaler.workers.should == 1}
35
- end
36
-
37
- describe 'yields' do
38
- it {client.call(Class, {}, 'queue') {:foo}.should == :foo}
39
- end
40
- end
41
-
42
- describe Autoscaler::Sidekiq::Server do
43
- let(:cut) {Autoscaler::Sidekiq::Server}
44
- let(:server) {cut.new(scaler, 0, ['queue'])}
45
- let(:workers) {1}
46
-
47
- def with_work_in_set(queue, set)
48
- payload = Sidekiq.dump_json('queue' => queue)
49
- Sidekiq.redis { |c| c.zadd(set, (Time.now.to_f + 30.to_f).to_s, payload)}
50
- end
51
-
52
- def with_scheduled_work_in(queue)
53
- with_work_in_set(queue, 'schedule')
54
- end
55
-
56
- def with_retry_work_in(queue)
57
- with_work_in_set(queue, 'retry')
58
- end
59
-
60
- def when_run
61
- server.call(Object.new, {}, 'queue') {}
62
- end
63
-
64
- def self.when_run_should_scale
65
- it('should downscale') do
66
- when_run
67
- scaler.workers.should == 0
68
- end
69
- end
70
-
71
- def self.when_run_should_not_scale
72
- it('should not downscale') do
73
- when_run
74
- scaler.workers.should == 1
75
- end
76
- end
77
-
78
- describe 'scales' do
79
- context "with no work" do
80
- before do
81
- server.stub(:sidekiq_queues).and_return({'queue' => 0, 'another_queue' => 1})
82
- end
83
- when_run_should_scale
84
- end
85
-
86
- context "with scheduled work in another queue" do
87
- before do
88
- with_scheduled_work_in('another_queue')
89
- end
90
- when_run_should_scale
91
- end
92
-
93
- context "with retry work in another queue" do
94
- before do
95
- with_retry_work_in('another_queue')
96
- end
97
- when_run_should_scale
98
- end
99
-
100
- context "when another process is working" do
101
- let(:other_process) {cut.new(Scaler.new(0), 10, ['other_queue'])}
102
- before do
103
- other_process.idle!('other_queue')
104
- server.working!('queue')
105
- end
106
- it {other_process.should be_idle}
107
- end
108
- end
109
-
110
- describe 'does not scale' do
111
- context "with enqueued work" do
112
- before do
113
- server.stub(:sidekiq_queues).and_return({'queue' => 1})
114
- end
115
- when_run_should_not_scale
116
- end
117
-
118
- context "with schedule work" do
119
- before do
120
- with_scheduled_work_in('queue')
121
- end
122
- when_run_should_not_scale
123
- end
124
-
125
- context "with retry work" do
126
- before do
127
- with_retry_work_in('queue')
128
- end
129
- when_run_should_not_scale
130
- end
131
- end
132
-
133
- describe 'yields' do
134
- it {server.call(Object.new, {}, 'queue') {:foo}.should == :foo}
135
- end
136
- end
137
- end