autoscaler 0.2.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGELOG.md +7 -1
- data/README.md +6 -8
- data/lib/autoscaler/sidekiq.rb +3 -106
- data/lib/autoscaler/sidekiq/activity.rb +51 -0
- data/lib/autoscaler/sidekiq/celluloid_monitor.rb +75 -0
- data/lib/autoscaler/sidekiq/client.rb +19 -0
- data/lib/autoscaler/sidekiq/monitor_middleware_adapter.rb +33 -0
- data/lib/autoscaler/sidekiq/queue_system.rb +72 -0
- data/lib/autoscaler/sidekiq/sleep_wait_server.rb +52 -0
- data/lib/autoscaler/stub_scaler.rb +25 -0
- data/lib/autoscaler/version.rb +1 -1
- data/spec/autoscaler/sidekiq/activity_spec.rb +21 -0
- data/spec/autoscaler/sidekiq/celluloid_monitor_spec.rb +38 -0
- data/spec/autoscaler/sidekiq/client_spec.rb +15 -0
- data/spec/autoscaler/sidekiq/monitor_middleware_adapter_spec.rb +15 -0
- data/spec/autoscaler/sidekiq/queue_system_spec.rb +63 -0
- data/spec/autoscaler/sidekiq/sleep_wait_server_spec.rb +31 -0
- data/spec/spec_helper.rb +8 -0
- metadata +27 -16
- data/spec/autoscaler/sidekiq_spec.rb +0 -137
data/CHANGELOG.md
CHANGED
@@ -1,9 +1,15 @@
|
|
1
1
|
# Changelog
|
2
2
|
|
3
|
+
## 0.3.0
|
4
|
+
|
5
|
+
- Downscale method changed from busy-waiting workers to a separate monitor process
|
6
|
+
- Minimum Sidekiq version raised to 2.7 to take advantage of Worker API
|
7
|
+
- Internal refactoring
|
8
|
+
- Autoscaler::StubScaler may be used for local testing
|
9
|
+
|
3
10
|
## 0.2.1
|
4
11
|
|
5
12
|
- Separate background activity flags to avoid crosstalk between processes
|
6
|
-
- Autoscaler::StubScaler may be used for local testing
|
7
13
|
|
8
14
|
## 0.2.0
|
9
15
|
|
data/README.md
CHANGED
@@ -19,6 +19,9 @@ This gem uses the [Herkou-Api](https://github.com/heroku/heroku.rb) gem, which r
|
|
19
19
|
|
20
20
|
Install the middleware in your `Sidekiq.configure_` blocks
|
21
21
|
|
22
|
+
require 'autoscaler/sidekiq'
|
23
|
+
require 'autoscaler/heroku_scaler'
|
24
|
+
|
22
25
|
Sidekiq.configure_client do |config|
|
23
26
|
config.client_middleware do |chain|
|
24
27
|
chain.add Autoscaler::Sidekiq::Client, 'default' => Autoscaler::HerokuScaler.new
|
@@ -33,17 +36,12 @@ Install the middleware in your `Sidekiq.configure_` blocks
|
|
33
36
|
|
34
37
|
## Limits and Challenges
|
35
38
|
|
36
|
-
- HerokuScaler includes an attempt at current-worker cache that may be overcomplication, and doesn't work very well
|
37
|
-
- Multiple
|
38
|
-
-
|
39
|
-
- If you set job-timeouts on your tasks, they will likely trigger on the sleep-loop (see previous).
|
39
|
+
- HerokuScaler includes an attempt at current-worker cache that may be overcomplication, and doesn't work very well on the server
|
40
|
+
- Multiple scale-down loops may be started, particularly if there are multiple jobs queued when the servers comes up. Heroku seems to handle multiple scale-down commands well.
|
41
|
+
- The scale-down monitor is triggered on job completion (and server middleware is only run around jobs), so if the server nevers processes any jobs, it won't turn off.
|
40
42
|
- The retry and schedule lists are considered - if you schedule a long-running task, the process will not scale-down.
|
41
43
|
- If background jobs trigger jobs in other scaled processes, please note you'll need `config.client_middleware` in your `Sidekiq.configure_server` block in order to scale-up.
|
42
44
|
|
43
|
-
### Long Jobs
|
44
|
-
|
45
|
-
Since the shutdown check gets performed every time a job completes, the shutdown-timeout will need to be longer than the longest job. For mixed workloads, you might want to have multiple sidekiq processes defined. I use one with many workers for general work, and a single-worker process for long import jobs. See `examples/complex.rb`
|
46
|
-
|
47
45
|
## Tests
|
48
46
|
|
49
47
|
The project is setup to run RSpec with Guard. It expects a redis instance on a custom port, which is started by the Guardfile.
|
data/lib/autoscaler/sidekiq.rb
CHANGED
@@ -1,114 +1,11 @@
|
|
1
|
-
require 'sidekiq'
|
1
|
+
require 'autoscaler/sidekiq/client'
|
2
|
+
require 'autoscaler/sidekiq/monitor_middleware_adapter'
|
2
3
|
|
3
4
|
module Autoscaler
|
4
5
|
# namespace module for Sidekiq middlewares
|
5
6
|
module Sidekiq
|
6
|
-
# Sidekiq client middleware
|
7
|
-
# Performs scale-up when items are queued and there are no workers running
|
8
|
-
class Client
|
9
|
-
# @param [Hash] scalers map of queue(String) => scaler (e.g. {HerokuScaler}).
|
10
|
-
# Which scaler to use for each sidekiq queue
|
11
|
-
def initialize(scalers)
|
12
|
-
@scalers = scalers
|
13
|
-
end
|
14
|
-
|
15
|
-
# Sidekiq middleware api method
|
16
|
-
def call(worker_class, item, queue)
|
17
|
-
@scalers[queue] && @scalers[queue].workers = 1
|
18
|
-
yield
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
7
|
# Sidekiq server middleware
|
23
8
|
# Performs scale-down when the queue is empty
|
24
|
-
|
25
|
-
# @param [scaler] scaler object that actually performs scaling operations (e.g. {HerokuScaler})
|
26
|
-
# @param [Numeric] timeout number of seconds to wait before shutdown
|
27
|
-
# @param [Array[String]] specified_queues list of queues to monitor to determine if there is work left. Defaults to all sidekiq queues.
|
28
|
-
def initialize(scaler, timeout, specified_queues = nil)
|
29
|
-
@scaler = scaler
|
30
|
-
@timeout = timeout
|
31
|
-
@specified_queues = specified_queues
|
32
|
-
end
|
33
|
-
|
34
|
-
# Sidekiq middleware api entry point
|
35
|
-
def call(worker, msg, queue)
|
36
|
-
working!(queue)
|
37
|
-
yield
|
38
|
-
ensure
|
39
|
-
working!(queue)
|
40
|
-
wait_for_task_or_scale
|
41
|
-
end
|
42
|
-
|
43
|
-
private
|
44
|
-
def refresh_sidekiq_queues!
|
45
|
-
@sidekiq_queues = ::Sidekiq::Stats.new.queues
|
46
|
-
end
|
47
|
-
|
48
|
-
attr_reader :sidekiq_queues
|
49
|
-
|
50
|
-
def queue_names
|
51
|
-
(@specified_queues || sidekiq_queues.keys)
|
52
|
-
end
|
53
|
-
|
54
|
-
def queued_work?
|
55
|
-
queue_names.any? {|name| sidekiq_queues[name].to_i > 0 }
|
56
|
-
end
|
57
|
-
|
58
|
-
def scheduled_work?
|
59
|
-
empty_sorted_set?("schedule")
|
60
|
-
end
|
61
|
-
|
62
|
-
def retry_work?
|
63
|
-
empty_sorted_set?("retry")
|
64
|
-
end
|
65
|
-
|
66
|
-
def empty_sorted_set?(sorted_set)
|
67
|
-
ss = ::Sidekiq::SortedSet.new(sorted_set)
|
68
|
-
ss.any? { |job| queue_names.include?(job.queue) }
|
69
|
-
end
|
70
|
-
|
71
|
-
def pending_work?
|
72
|
-
refresh_sidekiq_queues!
|
73
|
-
queued_work? || scheduled_work? || retry_work?
|
74
|
-
end
|
75
|
-
|
76
|
-
def wait_for_task_or_scale
|
77
|
-
loop do
|
78
|
-
return if pending_work?
|
79
|
-
return @scaler.workers = 0 if idle?
|
80
|
-
sleep(0.5)
|
81
|
-
end
|
82
|
-
end
|
83
|
-
|
84
|
-
def working!(queue)
|
85
|
-
active_at queue, Time.now
|
86
|
-
end
|
87
|
-
|
88
|
-
# test support
|
89
|
-
def idle!(queue)
|
90
|
-
active_at queue, Time.now - @timeout*2
|
91
|
-
end
|
92
|
-
|
93
|
-
def idle_time
|
94
|
-
t = last_activity
|
95
|
-
return 0 unless t
|
96
|
-
Time.now - Time.parse(t)
|
97
|
-
end
|
98
|
-
|
99
|
-
def idle?
|
100
|
-
idle_time > @timeout
|
101
|
-
end
|
102
|
-
|
103
|
-
def last_activity
|
104
|
-
::Sidekiq.redis {|c|
|
105
|
-
queue_names.map {|q| c.get('background_activity:'+q)}.compact.max
|
106
|
-
}
|
107
|
-
end
|
108
|
-
|
109
|
-
def active_at(queue, time)
|
110
|
-
::Sidekiq.redis {|c| c.set('background_activity:'+queue, time)}
|
111
|
-
end
|
112
|
-
end
|
9
|
+
Server = MonitorMiddlewareAdapter
|
113
10
|
end
|
114
11
|
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
require 'sidekiq'
|
2
|
+
|
3
|
+
module Autoscaler
|
4
|
+
module Sidekiq
|
5
|
+
# Tracks activity timeouts using Sidekiq's redis connection
|
6
|
+
class Activity
|
7
|
+
# @param [Numeric] timeout number of seconds to wait before shutdown
|
8
|
+
def initialize(timeout)
|
9
|
+
@timeout = timeout
|
10
|
+
end
|
11
|
+
|
12
|
+
# Record that a queue has activity
|
13
|
+
# @param [String] queue
|
14
|
+
def working!(queue)
|
15
|
+
active_at queue, Time.now
|
16
|
+
end
|
17
|
+
|
18
|
+
# Record that a queue is idle and timed out - mostly for test support
|
19
|
+
# @param [String] queue
|
20
|
+
def idle!(queue)
|
21
|
+
active_at queue, Time.now - timeout*2
|
22
|
+
end
|
23
|
+
|
24
|
+
# Have the watched queues timed out?
|
25
|
+
# @param [Array[String]] queues list of queues to monitor to determine if there is work left
|
26
|
+
# @return [boolean]
|
27
|
+
def idle?(queues)
|
28
|
+
idle_time(queues) > timeout
|
29
|
+
end
|
30
|
+
|
31
|
+
private
|
32
|
+
attr_reader :timeout
|
33
|
+
|
34
|
+
def idle_time(queues)
|
35
|
+
t = last_activity(queues)
|
36
|
+
return 0 unless t
|
37
|
+
Time.now - Time.parse(t)
|
38
|
+
end
|
39
|
+
|
40
|
+
def last_activity(queues)
|
41
|
+
::Sidekiq.redis {|c|
|
42
|
+
queues.map {|q| c.get('background_activity:'+q)}.compact.max
|
43
|
+
}
|
44
|
+
end
|
45
|
+
|
46
|
+
def active_at(queue, time)
|
47
|
+
::Sidekiq.redis {|c| c.set('background_activity:'+queue, time)}
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
@@ -0,0 +1,75 @@
|
|
1
|
+
require 'celluloid'
|
2
|
+
|
3
|
+
module Autoscaler
|
4
|
+
module Sidekiq
|
5
|
+
# Actor to monitor the sidekiq server for scale-down
|
6
|
+
class CelluloidMonitor
|
7
|
+
include Celluloid
|
8
|
+
|
9
|
+
# @param [scaler] scaler object that actually performs scaling operations (e.g. {HerokuScaler})
|
10
|
+
# @param [Numeric] timeout number of seconds to wait before shutdown
|
11
|
+
# @param [System] system interface to the queuing system that provides `pending_work?`
|
12
|
+
def initialize(scaler, timeout, system)
|
13
|
+
@scaler = scaler
|
14
|
+
@timeout = timeout
|
15
|
+
@poll = [timeout/4.0, 0.5].max
|
16
|
+
@system = system
|
17
|
+
@running = false
|
18
|
+
end
|
19
|
+
|
20
|
+
# Mostly sleep until there has been no activity for the timeout
|
21
|
+
def wait_for_downscale
|
22
|
+
once do
|
23
|
+
active_now!
|
24
|
+
|
25
|
+
while active? || time_left?
|
26
|
+
sleep(@poll)
|
27
|
+
update_activity
|
28
|
+
end
|
29
|
+
|
30
|
+
@scaler.workers = 0
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
# Notify the monitor that a job is starting
|
35
|
+
def starting_job
|
36
|
+
end
|
37
|
+
|
38
|
+
# Notify the monitor that a job has finished
|
39
|
+
def finished_job
|
40
|
+
active_now!
|
41
|
+
async.wait_for_downscale
|
42
|
+
end
|
43
|
+
|
44
|
+
private
|
45
|
+
attr_reader :system
|
46
|
+
|
47
|
+
def active?
|
48
|
+
system.pending_work? || system.working?
|
49
|
+
end
|
50
|
+
|
51
|
+
def update_activity
|
52
|
+
active_now! if active?
|
53
|
+
end
|
54
|
+
|
55
|
+
def active_now!
|
56
|
+
@activity = Time.now
|
57
|
+
end
|
58
|
+
|
59
|
+
def time_left?
|
60
|
+
(Time.now - @activity) < @timeout
|
61
|
+
end
|
62
|
+
|
63
|
+
def once
|
64
|
+
return if @running
|
65
|
+
|
66
|
+
begin
|
67
|
+
@running = true
|
68
|
+
yield
|
69
|
+
ensure
|
70
|
+
@running = false
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
module Autoscaler
|
2
|
+
module Sidekiq
|
3
|
+
# Sidekiq client middleware
|
4
|
+
# Performs scale-up when items are queued and there are no workers running
|
5
|
+
class Client
|
6
|
+
# @param [Hash] scalers map of queue(String) => scaler (e.g. {HerokuScaler}).
|
7
|
+
# Which scaler to use for each sidekiq queue
|
8
|
+
def initialize(scalers)
|
9
|
+
@scalers = scalers
|
10
|
+
end
|
11
|
+
|
12
|
+
# Sidekiq middleware api method
|
13
|
+
def call(worker_class, item, queue)
|
14
|
+
@scalers[queue] && @scalers[queue].workers = 1
|
15
|
+
yield
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
require 'autoscaler/sidekiq/queue_system'
|
2
|
+
require 'autoscaler/sidekiq/celluloid_monitor'
|
3
|
+
|
4
|
+
module Autoscaler
|
5
|
+
module Sidekiq
|
6
|
+
# Shim to the existing autoscaler interface
|
7
|
+
# Starts the monitor and notifies it of job events that may occur while it's sleeping
|
8
|
+
class MonitorMiddlewareAdapter
|
9
|
+
# @param [scaler] scaler object that actually performs scaling operations (e.g. {HerokuScaler})
|
10
|
+
# @param [Numeric] timeout number of seconds to wait before shutdown
|
11
|
+
# @param [Array[String]] specified_queues list of queues to monitor to determine if there is work left. Defaults to all sidekiq queues.
|
12
|
+
def initialize(scaler, timeout, specified_queues = nil)
|
13
|
+
system = QueueSystem.new(specified_queues)
|
14
|
+
unless monitor
|
15
|
+
CelluloidMonitor.supervise_as(:autoscaler_monitor, scaler, timeout, system)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
# Sidekiq middleware api entry point
|
20
|
+
def call(worker, msg, queue)
|
21
|
+
monitor.async.starting_job
|
22
|
+
yield
|
23
|
+
ensure
|
24
|
+
monitor.async.finished_job
|
25
|
+
end
|
26
|
+
|
27
|
+
private
|
28
|
+
def monitor
|
29
|
+
Celluloid::Actor[:autoscaler_monitor]
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,72 @@
|
|
1
|
+
require 'sidekiq/api'
|
2
|
+
|
3
|
+
module Autoscaler
|
4
|
+
module Sidekiq
|
5
|
+
# Interface to to interrogate the queuing system
|
6
|
+
class QueueSystem
|
7
|
+
# @param [Array[String]] specified_queues list of queues to monitor to determine if there is work left. Defaults to all sidekiq queues.
|
8
|
+
def initialize(specified_queues = nil)
|
9
|
+
@specified_queues = specified_queues
|
10
|
+
end
|
11
|
+
|
12
|
+
# @return [boolean] whether there is queued work - does not include work currently in progress
|
13
|
+
def pending_work?
|
14
|
+
refresh_sidekiq_queues!
|
15
|
+
queued_work? || scheduled_work? || retry_work?
|
16
|
+
end
|
17
|
+
|
18
|
+
# @return [boolean] whether there are workers actively running
|
19
|
+
def working?
|
20
|
+
workers > 0
|
21
|
+
end
|
22
|
+
|
23
|
+
# @return [Array[String]]
|
24
|
+
def queue_names
|
25
|
+
(@specified_queues || sidekiq_queues.keys)
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
def queued_work?
|
30
|
+
queue_names.any? {|name| sidekiq_queues[name].to_i > 0 }
|
31
|
+
end
|
32
|
+
|
33
|
+
def scheduled_work?
|
34
|
+
empty_sorted_set?("schedule")
|
35
|
+
end
|
36
|
+
|
37
|
+
def retry_work?
|
38
|
+
empty_sorted_set?("retry")
|
39
|
+
end
|
40
|
+
|
41
|
+
attr_reader :sidekiq_queues
|
42
|
+
|
43
|
+
def refresh_sidekiq_queues!
|
44
|
+
@sidekiq_queues = ::Sidekiq::Stats.new.queues
|
45
|
+
end
|
46
|
+
|
47
|
+
def empty_sorted_set?(sorted_set)
|
48
|
+
ss = ::Sidekiq::SortedSet.new(sorted_set)
|
49
|
+
ss.any? { |job| queue_names.include?(job.queue) }
|
50
|
+
end
|
51
|
+
|
52
|
+
def workers
|
53
|
+
if @specified_queues
|
54
|
+
specified_workers
|
55
|
+
else
|
56
|
+
total_workers
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
def total_workers
|
61
|
+
::Sidekiq::Workers.new.size
|
62
|
+
end
|
63
|
+
|
64
|
+
def specified_workers
|
65
|
+
refresh_sidekiq_queues!
|
66
|
+
::Sidekiq::Workers.new.count {|name, work|
|
67
|
+
queue_names.include?(work['queue'])
|
68
|
+
}
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
@@ -0,0 +1,52 @@
|
|
1
|
+
require 'autoscaler/sidekiq/queue_system'
|
2
|
+
require 'autoscaler/sidekiq/activity'
|
3
|
+
|
4
|
+
module Autoscaler
|
5
|
+
module Sidekiq
|
6
|
+
# Sidekiq server middleware
|
7
|
+
# Performs scale-down when the queue is empty
|
8
|
+
class SleepWaitServer
|
9
|
+
# @param [scaler] scaler object that actually performs scaling operations (e.g. {HerokuScaler})
|
10
|
+
# @param [Numeric] timeout number of seconds to wait before shutdown
|
11
|
+
# @param [Array[String]] specified_queues list of queues to monitor to determine if there is work left. Defaults to all sidekiq queues.
|
12
|
+
def initialize(scaler, timeout, specified_queues = nil)
|
13
|
+
@scaler = scaler
|
14
|
+
@activity = Activity.new(timeout)
|
15
|
+
@system = QueueSystem.new(specified_queues)
|
16
|
+
end
|
17
|
+
|
18
|
+
# Sidekiq middleware api entry point
|
19
|
+
def call(worker, msg, queue)
|
20
|
+
working!(queue)
|
21
|
+
yield
|
22
|
+
ensure
|
23
|
+
working!(queue)
|
24
|
+
wait_for_task_or_scale
|
25
|
+
end
|
26
|
+
|
27
|
+
private
|
28
|
+
def wait_for_task_or_scale
|
29
|
+
loop do
|
30
|
+
return if pending_work?
|
31
|
+
return @scaler.workers = 0 if idle?
|
32
|
+
sleep(0.5)
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
attr_reader :system
|
37
|
+
attr_reader :activity
|
38
|
+
|
39
|
+
def pending_work?
|
40
|
+
system.pending_work?
|
41
|
+
end
|
42
|
+
|
43
|
+
def working!(queue)
|
44
|
+
activity.working!(queue)
|
45
|
+
end
|
46
|
+
|
47
|
+
def idle?
|
48
|
+
activity.idle?(system.queue_names)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require 'heroku-api'
|
2
|
+
|
3
|
+
module Autoscaler
|
4
|
+
# A minimal scaler to use as stub for local testing
|
5
|
+
class StubScaler
|
6
|
+
# @param [String] type used to distinguish messages from multiple stubs
|
7
|
+
def initialize(type = 'worker')
|
8
|
+
@type = type
|
9
|
+
@workers = 0
|
10
|
+
end
|
11
|
+
|
12
|
+
attr_reader :type
|
13
|
+
|
14
|
+
# Read the current worker count
|
15
|
+
# @return [Numeric] number of workers
|
16
|
+
attr_reader :workers
|
17
|
+
|
18
|
+
# Set the number of workers
|
19
|
+
# @param [Numeric] n number of workers
|
20
|
+
def workers=(n)
|
21
|
+
p "Scaling #{type} to #{n}"
|
22
|
+
@workers = n
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
data/lib/autoscaler/version.rb
CHANGED
@@ -0,0 +1,21 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'autoscaler/sidekiq/activity'
|
3
|
+
|
4
|
+
describe Autoscaler::Sidekiq::Activity do
|
5
|
+
before do
|
6
|
+
@redis = Sidekiq.redis = REDIS
|
7
|
+
Sidekiq.redis {|c| c.flushdb }
|
8
|
+
end
|
9
|
+
|
10
|
+
let(:cut) {Autoscaler::Sidekiq::Activity}
|
11
|
+
let(:activity) {cut.new(0)}
|
12
|
+
|
13
|
+
context "when another process is working" do
|
14
|
+
let(:other_process) {cut.new(10)}
|
15
|
+
before do
|
16
|
+
activity.idle!('queue')
|
17
|
+
other_process.working!('other_queue')
|
18
|
+
end
|
19
|
+
it {activity.should be_idle(['queue'])}
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'autoscaler/sidekiq/celluloid_monitor'
|
3
|
+
require 'timeout'
|
4
|
+
|
5
|
+
class TestSystem
|
6
|
+
def initialize(pending)
|
7
|
+
@pending = pending
|
8
|
+
end
|
9
|
+
|
10
|
+
def working?; false; end
|
11
|
+
def pending_work?; @pending; end
|
12
|
+
end
|
13
|
+
|
14
|
+
describe Autoscaler::Sidekiq::CelluloidMonitor do
|
15
|
+
before do
|
16
|
+
@redis = Sidekiq.redis = REDIS
|
17
|
+
Sidekiq.redis {|c| c.flushdb }
|
18
|
+
end
|
19
|
+
|
20
|
+
let(:cut) {Autoscaler::Sidekiq::CelluloidMonitor}
|
21
|
+
let(:scaler) {TestScaler.new(1)}
|
22
|
+
|
23
|
+
it "scales with no work" do
|
24
|
+
system = TestSystem.new(false)
|
25
|
+
manager = cut.new(scaler, 0, system)
|
26
|
+
Timeout.timeout(1) { manager.wait_for_downscale }
|
27
|
+
scaler.workers.should == 0
|
28
|
+
manager.terminate
|
29
|
+
end
|
30
|
+
|
31
|
+
it "does not scale with pending work" do
|
32
|
+
system = TestSystem.new(true)
|
33
|
+
manager = cut.new(scaler, 0, system)
|
34
|
+
expect {Timeout.timeout(1) { manager.wait_for_downscale }}.to raise_error Timeout::Error
|
35
|
+
scaler.workers.should == 1
|
36
|
+
manager.terminate
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'autoscaler/sidekiq/client'
|
3
|
+
|
4
|
+
describe Autoscaler::Sidekiq::Client do
|
5
|
+
let(:cut) {Autoscaler::Sidekiq::Client}
|
6
|
+
let(:scaler) {TestScaler.new(0)}
|
7
|
+
let(:client) {cut.new('queue' => scaler)}
|
8
|
+
|
9
|
+
it 'scales' do
|
10
|
+
client.call(Class, {}, 'queue') {}
|
11
|
+
scaler.workers.should == 1
|
12
|
+
end
|
13
|
+
|
14
|
+
it('yields') {client.call(Class, {}, 'queue') {:foo}.should == :foo}
|
15
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'autoscaler/sidekiq/monitor_middleware_adapter'
|
3
|
+
|
4
|
+
describe Autoscaler::Sidekiq::MonitorMiddlewareAdapter do
|
5
|
+
before do
|
6
|
+
@redis = Sidekiq.redis = REDIS
|
7
|
+
Sidekiq.redis {|c| c.flushdb }
|
8
|
+
end
|
9
|
+
|
10
|
+
let(:cut) {Autoscaler::Sidekiq::MonitorMiddlewareAdapter}
|
11
|
+
let(:scaler) {TestScaler.new(1)}
|
12
|
+
let(:server) {cut.new(scaler, 0, ['queue'])}
|
13
|
+
|
14
|
+
it('yields') {server.call(Object.new, {}, 'queue') {:foo}.should == :foo}
|
15
|
+
end
|
@@ -0,0 +1,63 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'autoscaler/sidekiq/queue_system'
|
3
|
+
|
4
|
+
describe Autoscaler::Sidekiq::QueueSystem do
|
5
|
+
let(:cut) {Autoscaler::Sidekiq::QueueSystem}
|
6
|
+
|
7
|
+
before do
|
8
|
+
@redis = Sidekiq.redis = REDIS
|
9
|
+
Sidekiq.redis {|c| c.flushdb }
|
10
|
+
end
|
11
|
+
|
12
|
+
def with_work_in_set(queue, set)
|
13
|
+
payload = Sidekiq.dump_json('queue' => queue)
|
14
|
+
Sidekiq.redis { |c| c.zadd(set, (Time.now.to_f + 30.to_f).to_s, payload)}
|
15
|
+
end
|
16
|
+
|
17
|
+
def with_scheduled_work_in(queue)
|
18
|
+
with_work_in_set(queue, 'schedule')
|
19
|
+
end
|
20
|
+
|
21
|
+
def with_retry_work_in(queue)
|
22
|
+
with_work_in_set(queue, 'retry')
|
23
|
+
end
|
24
|
+
|
25
|
+
subject {cut.new(['queue'])}
|
26
|
+
|
27
|
+
it {subject.queue_names.should == ['queue']}
|
28
|
+
it {subject.working?.should be_false}
|
29
|
+
|
30
|
+
describe 'no pending work' do
|
31
|
+
it "with no work" do
|
32
|
+
subject.stub(:sidekiq_queues).and_return({'queue' => 0, 'another_queue' => 1})
|
33
|
+
subject.should_not be_pending_work
|
34
|
+
end
|
35
|
+
|
36
|
+
it "with scheduled work in another queue" do
|
37
|
+
with_scheduled_work_in('another_queue')
|
38
|
+
subject.should_not be_pending_work
|
39
|
+
end
|
40
|
+
|
41
|
+
it "with retry work in another queue" do
|
42
|
+
with_retry_work_in('another_queue')
|
43
|
+
subject.should_not be_pending_work
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
describe 'with pending work' do
|
48
|
+
it "with enqueued work" do
|
49
|
+
subject.stub(:sidekiq_queues).and_return({'queue' => 1})
|
50
|
+
subject.should be_pending_work
|
51
|
+
end
|
52
|
+
|
53
|
+
it "with schedule work" do
|
54
|
+
with_scheduled_work_in('queue')
|
55
|
+
subject.should be_pending_work
|
56
|
+
end
|
57
|
+
|
58
|
+
it "with retry work" do
|
59
|
+
with_retry_work_in('queue')
|
60
|
+
subject.should be_pending_work
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'autoscaler/sidekiq/sleep_wait_server'
|
3
|
+
|
4
|
+
describe Autoscaler::Sidekiq::SleepWaitServer do
|
5
|
+
before do
|
6
|
+
@redis = Sidekiq.redis = REDIS
|
7
|
+
Sidekiq.redis {|c| c.flushdb }
|
8
|
+
end
|
9
|
+
|
10
|
+
let(:cut) {Autoscaler::Sidekiq::SleepWaitServer}
|
11
|
+
let(:scaler) {TestScaler.new(1)}
|
12
|
+
let(:server) {cut.new(scaler, 0, ['queue'])}
|
13
|
+
|
14
|
+
def when_run
|
15
|
+
server.call(Object.new, {}, 'queue') {}
|
16
|
+
end
|
17
|
+
|
18
|
+
it "scales with no work" do
|
19
|
+
server.stub(:pending_work?).and_return(false)
|
20
|
+
when_run
|
21
|
+
scaler.workers.should == 0
|
22
|
+
end
|
23
|
+
|
24
|
+
it "does not scale with pending work" do
|
25
|
+
server.stub(:pending_work?).and_return(true)
|
26
|
+
when_run
|
27
|
+
scaler.workers.should == 1
|
28
|
+
end
|
29
|
+
|
30
|
+
it('yields') {server.call(Object.new, {}, 'queue') {:foo}.should == :foo}
|
31
|
+
end
|
data/spec/spec_helper.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: autoscaler
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -10,30 +10,24 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2013-
|
13
|
+
date: 2013-05-18 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: sidekiq
|
17
17
|
requirement: !ruby/object:Gem::Requirement
|
18
18
|
none: false
|
19
19
|
requirements:
|
20
|
-
- -
|
21
|
-
- !ruby/object:Gem::Version
|
22
|
-
version: 2.6.1
|
23
|
-
- - <
|
20
|
+
- - ~>
|
24
21
|
- !ruby/object:Gem::Version
|
25
|
-
version: '
|
22
|
+
version: '2.7'
|
26
23
|
type: :runtime
|
27
24
|
prerelease: false
|
28
25
|
version_requirements: !ruby/object:Gem::Requirement
|
29
26
|
none: false
|
30
27
|
requirements:
|
31
|
-
- -
|
32
|
-
- !ruby/object:Gem::Version
|
33
|
-
version: 2.6.1
|
34
|
-
- - <
|
28
|
+
- - ~>
|
35
29
|
- !ruby/object:Gem::Version
|
36
|
-
version: '
|
30
|
+
version: '2.7'
|
37
31
|
- !ruby/object:Gem::Dependency
|
38
32
|
name: heroku-api
|
39
33
|
requirement: !ruby/object:Gem::Requirement
|
@@ -139,7 +133,14 @@ extensions: []
|
|
139
133
|
extra_rdoc_files: []
|
140
134
|
files:
|
141
135
|
- lib/autoscaler/heroku_scaler.rb
|
136
|
+
- lib/autoscaler/sidekiq/activity.rb
|
137
|
+
- lib/autoscaler/sidekiq/celluloid_monitor.rb
|
138
|
+
- lib/autoscaler/sidekiq/client.rb
|
139
|
+
- lib/autoscaler/sidekiq/monitor_middleware_adapter.rb
|
140
|
+
- lib/autoscaler/sidekiq/queue_system.rb
|
141
|
+
- lib/autoscaler/sidekiq/sleep_wait_server.rb
|
142
142
|
- lib/autoscaler/sidekiq.rb
|
143
|
+
- lib/autoscaler/stub_scaler.rb
|
143
144
|
- lib/autoscaler/version.rb
|
144
145
|
- lib/autoscaler.rb
|
145
146
|
- README.md
|
@@ -148,9 +149,14 @@ files:
|
|
148
149
|
- examples/simple.rb
|
149
150
|
- Guardfile
|
150
151
|
- spec/autoscaler/heroku_scaler_spec.rb
|
151
|
-
- spec/autoscaler/
|
152
|
-
- spec/
|
152
|
+
- spec/autoscaler/sidekiq/activity_spec.rb
|
153
|
+
- spec/autoscaler/sidekiq/celluloid_monitor_spec.rb
|
154
|
+
- spec/autoscaler/sidekiq/client_spec.rb
|
155
|
+
- spec/autoscaler/sidekiq/monitor_middleware_adapter_spec.rb
|
156
|
+
- spec/autoscaler/sidekiq/queue_system_spec.rb
|
157
|
+
- spec/autoscaler/sidekiq/sleep_wait_server_spec.rb
|
153
158
|
- spec/redis_test.conf
|
159
|
+
- spec/spec_helper.rb
|
154
160
|
homepage: ''
|
155
161
|
licenses: []
|
156
162
|
post_install_message:
|
@@ -178,7 +184,12 @@ summary: Start/stop Sidekiq workers on Heroku
|
|
178
184
|
test_files:
|
179
185
|
- Guardfile
|
180
186
|
- spec/autoscaler/heroku_scaler_spec.rb
|
181
|
-
- spec/autoscaler/
|
182
|
-
- spec/
|
187
|
+
- spec/autoscaler/sidekiq/activity_spec.rb
|
188
|
+
- spec/autoscaler/sidekiq/celluloid_monitor_spec.rb
|
189
|
+
- spec/autoscaler/sidekiq/client_spec.rb
|
190
|
+
- spec/autoscaler/sidekiq/monitor_middleware_adapter_spec.rb
|
191
|
+
- spec/autoscaler/sidekiq/queue_system_spec.rb
|
192
|
+
- spec/autoscaler/sidekiq/sleep_wait_server_spec.rb
|
183
193
|
- spec/redis_test.conf
|
194
|
+
- spec/spec_helper.rb
|
184
195
|
has_rdoc:
|
@@ -1,137 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
require 'autoscaler/sidekiq'
|
3
|
-
|
4
|
-
class Scaler
|
5
|
-
attr_accessor :workers
|
6
|
-
|
7
|
-
def initialize(n = 0)
|
8
|
-
self.workers = n
|
9
|
-
end
|
10
|
-
end
|
11
|
-
|
12
|
-
class Autoscaler::Sidekiq::Server
|
13
|
-
public :idle!
|
14
|
-
public :working!
|
15
|
-
end
|
16
|
-
|
17
|
-
describe Autoscaler::Sidekiq do
|
18
|
-
before do
|
19
|
-
@redis = Sidekiq.redis = REDIS
|
20
|
-
Sidekiq.redis {|c| c.flushdb }
|
21
|
-
end
|
22
|
-
|
23
|
-
let(:scaler) do
|
24
|
-
Scaler.new(workers)
|
25
|
-
end
|
26
|
-
|
27
|
-
describe Autoscaler::Sidekiq::Client do
|
28
|
-
let(:cut) {Autoscaler::Sidekiq::Client}
|
29
|
-
let(:client) {cut.new('queue' => scaler)}
|
30
|
-
let(:workers) {0}
|
31
|
-
|
32
|
-
describe 'scales' do
|
33
|
-
before {client.call(Class, {}, 'queue') {}}
|
34
|
-
it {scaler.workers.should == 1}
|
35
|
-
end
|
36
|
-
|
37
|
-
describe 'yields' do
|
38
|
-
it {client.call(Class, {}, 'queue') {:foo}.should == :foo}
|
39
|
-
end
|
40
|
-
end
|
41
|
-
|
42
|
-
describe Autoscaler::Sidekiq::Server do
|
43
|
-
let(:cut) {Autoscaler::Sidekiq::Server}
|
44
|
-
let(:server) {cut.new(scaler, 0, ['queue'])}
|
45
|
-
let(:workers) {1}
|
46
|
-
|
47
|
-
def with_work_in_set(queue, set)
|
48
|
-
payload = Sidekiq.dump_json('queue' => queue)
|
49
|
-
Sidekiq.redis { |c| c.zadd(set, (Time.now.to_f + 30.to_f).to_s, payload)}
|
50
|
-
end
|
51
|
-
|
52
|
-
def with_scheduled_work_in(queue)
|
53
|
-
with_work_in_set(queue, 'schedule')
|
54
|
-
end
|
55
|
-
|
56
|
-
def with_retry_work_in(queue)
|
57
|
-
with_work_in_set(queue, 'retry')
|
58
|
-
end
|
59
|
-
|
60
|
-
def when_run
|
61
|
-
server.call(Object.new, {}, 'queue') {}
|
62
|
-
end
|
63
|
-
|
64
|
-
def self.when_run_should_scale
|
65
|
-
it('should downscale') do
|
66
|
-
when_run
|
67
|
-
scaler.workers.should == 0
|
68
|
-
end
|
69
|
-
end
|
70
|
-
|
71
|
-
def self.when_run_should_not_scale
|
72
|
-
it('should not downscale') do
|
73
|
-
when_run
|
74
|
-
scaler.workers.should == 1
|
75
|
-
end
|
76
|
-
end
|
77
|
-
|
78
|
-
describe 'scales' do
|
79
|
-
context "with no work" do
|
80
|
-
before do
|
81
|
-
server.stub(:sidekiq_queues).and_return({'queue' => 0, 'another_queue' => 1})
|
82
|
-
end
|
83
|
-
when_run_should_scale
|
84
|
-
end
|
85
|
-
|
86
|
-
context "with scheduled work in another queue" do
|
87
|
-
before do
|
88
|
-
with_scheduled_work_in('another_queue')
|
89
|
-
end
|
90
|
-
when_run_should_scale
|
91
|
-
end
|
92
|
-
|
93
|
-
context "with retry work in another queue" do
|
94
|
-
before do
|
95
|
-
with_retry_work_in('another_queue')
|
96
|
-
end
|
97
|
-
when_run_should_scale
|
98
|
-
end
|
99
|
-
|
100
|
-
context "when another process is working" do
|
101
|
-
let(:other_process) {cut.new(Scaler.new(0), 10, ['other_queue'])}
|
102
|
-
before do
|
103
|
-
other_process.idle!('other_queue')
|
104
|
-
server.working!('queue')
|
105
|
-
end
|
106
|
-
it {other_process.should be_idle}
|
107
|
-
end
|
108
|
-
end
|
109
|
-
|
110
|
-
describe 'does not scale' do
|
111
|
-
context "with enqueued work" do
|
112
|
-
before do
|
113
|
-
server.stub(:sidekiq_queues).and_return({'queue' => 1})
|
114
|
-
end
|
115
|
-
when_run_should_not_scale
|
116
|
-
end
|
117
|
-
|
118
|
-
context "with schedule work" do
|
119
|
-
before do
|
120
|
-
with_scheduled_work_in('queue')
|
121
|
-
end
|
122
|
-
when_run_should_not_scale
|
123
|
-
end
|
124
|
-
|
125
|
-
context "with retry work" do
|
126
|
-
before do
|
127
|
-
with_retry_work_in('queue')
|
128
|
-
end
|
129
|
-
when_run_should_not_scale
|
130
|
-
end
|
131
|
-
end
|
132
|
-
|
133
|
-
describe 'yields' do
|
134
|
-
it {server.call(Object.new, {}, 'queue') {:foo}.should == :foo}
|
135
|
-
end
|
136
|
-
end
|
137
|
-
end
|