resque-heroku-scaler 0.3.2 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/resque/plugins/heroku_scaler.rb +48 -27
- data/lib/resque/plugins/heroku_scaler/config.rb +1 -1
- data/lib/resque/plugins/heroku_scaler/resque.rb +16 -7
- data/lib/resque/plugins/heroku_scaler/version.rb +1 -1
- data/lib/resque/plugins/heroku_scaler/worker.rb +44 -36
- data/test/config_test.rb +2 -2
- data/test/heroku_scaler_test.rb +14 -6
- data/test/worker_test.rb +6 -17
- metadata +8 -8
@@ -29,30 +29,21 @@ module Resque
|
|
29
29
|
|
30
30
|
return if required == active
|
31
31
|
|
32
|
-
log "Scale workers from #{active} to #{required}"
|
33
|
-
|
34
32
|
if required > active
|
33
|
+
log "Scale workers from #{active} to #{required}"
|
35
34
|
scale_workers(required)
|
36
35
|
return
|
37
36
|
end
|
37
|
+
|
38
|
+
return if pending?
|
38
39
|
|
39
|
-
|
40
|
-
stop = timeout
|
41
|
-
wait_for_workers until ready_to_scale(active) or timeout?(stop)
|
42
|
-
scale_workers(required)
|
43
|
-
|
44
|
-
ensure
|
45
|
-
resume_workers
|
40
|
+
scale_down(active)
|
46
41
|
end
|
47
42
|
|
48
43
|
def wait_for_scale
|
49
44
|
sleep Resque::Plugins::HerokuScaler::Config.scale_interval
|
50
45
|
end
|
51
46
|
|
52
|
-
def wait_for_workers
|
53
|
-
sleep Resque::Plugins::HerokuScaler::Config.poll_interval
|
54
|
-
end
|
55
|
-
|
56
47
|
def scale_for(pending)
|
57
48
|
Resque::Plugins::HerokuScaler::Config.scale_for(pending)
|
58
49
|
end
|
@@ -61,32 +52,62 @@ module Resque
|
|
61
52
|
Resque::Plugins::HerokuScaler::Manager.workers = qty
|
62
53
|
end
|
63
54
|
|
55
|
+
def scale_down(active)
|
56
|
+
log "Scale #{active} workers down"
|
57
|
+
|
58
|
+
lock
|
59
|
+
|
60
|
+
timeout = Time.now + Resque::Plugins::HerokuScaler::Config.scale_timeout
|
61
|
+
until locked == active or Time.now >= timeout
|
62
|
+
sleep Resque::Plugins::HerokuScaler::Config.poll_interval
|
63
|
+
end
|
64
|
+
|
65
|
+
scale_workers(0)
|
66
|
+
|
67
|
+
timeout = Time.now + Resque::Plugins::HerokuScaler::Config.scale_timeout
|
68
|
+
until Time.now >= timeout
|
69
|
+
if offline?
|
70
|
+
log "#{active} workers scaled down successfully"
|
71
|
+
prune
|
72
|
+
break
|
73
|
+
end
|
74
|
+
sleep Resque::Plugins::HerokuScaler::Config.poll_interval
|
75
|
+
end
|
76
|
+
|
77
|
+
ensure
|
78
|
+
unlock
|
79
|
+
end
|
80
|
+
|
64
81
|
def workers
|
65
82
|
Resque::Plugins::HerokuScaler::Manager.workers
|
66
83
|
end
|
67
|
-
|
68
|
-
def
|
69
|
-
|
84
|
+
|
85
|
+
def offline?
|
86
|
+
workers.zero?
|
70
87
|
end
|
71
88
|
|
72
|
-
def
|
73
|
-
|
89
|
+
def pending?
|
90
|
+
pending > 0
|
74
91
|
end
|
75
92
|
|
76
|
-
def
|
77
|
-
|
93
|
+
def pending
|
94
|
+
Resque.info[:pending]
|
78
95
|
end
|
79
96
|
|
80
|
-
def
|
81
|
-
|
97
|
+
def lock
|
98
|
+
Resque.lock
|
82
99
|
end
|
83
100
|
|
84
|
-
def
|
85
|
-
Resque.
|
101
|
+
def unlock
|
102
|
+
Resque.unlock
|
86
103
|
end
|
87
|
-
|
88
|
-
def
|
89
|
-
Resque.info[:
|
104
|
+
|
105
|
+
def locked
|
106
|
+
Resque.info[:locked]
|
107
|
+
end
|
108
|
+
|
109
|
+
def prune
|
110
|
+
Resque.prune
|
90
111
|
end
|
91
112
|
|
92
113
|
def configure
|
@@ -1,14 +1,23 @@
|
|
1
1
|
module Resque
|
2
2
|
|
3
3
|
alias_method :original_info, :info
|
4
|
-
|
5
|
-
def scaling
|
6
|
-
Worker.scaling
|
7
|
-
end
|
8
4
|
|
9
5
|
def info
|
10
|
-
|
11
|
-
|
12
|
-
return
|
6
|
+
info_with_locked = original_info
|
7
|
+
info_with_locked[:locked] = Worker.locked.size
|
8
|
+
return info_with_locked
|
13
9
|
end
|
10
|
+
|
11
|
+
def lock
|
12
|
+
Worker.lock
|
13
|
+
end
|
14
|
+
|
15
|
+
def unlock
|
16
|
+
Worker.unlock
|
17
|
+
end
|
18
|
+
|
19
|
+
def prune
|
20
|
+
Worker.prune
|
21
|
+
end
|
22
|
+
|
14
23
|
end
|
@@ -1,18 +1,23 @@
|
|
1
1
|
module Resque
|
2
|
-
|
2
|
+
|
3
3
|
class Worker
|
4
|
-
|
5
|
-
|
4
|
+
|
6
5
|
def work(interval = 5.0, &block)
|
7
6
|
interval = Float(interval)
|
8
7
|
$0 = "resque: Starting"
|
9
8
|
startup
|
10
9
|
|
11
10
|
loop do
|
12
|
-
wait_for_scale if scaling?
|
13
11
|
break if shutdown?
|
12
|
+
|
13
|
+
if should_lock?
|
14
|
+
lock
|
15
|
+
break
|
16
|
+
end
|
17
|
+
|
18
|
+
pause if should_pause?
|
14
19
|
|
15
|
-
if
|
20
|
+
if job = reserve(interval)
|
16
21
|
log "got: #{job.inspect}"
|
17
22
|
job.worker = self
|
18
23
|
run_hook :before_fork, job
|
@@ -32,54 +37,57 @@ module Resque
|
|
32
37
|
@child = nil
|
33
38
|
else
|
34
39
|
break if interval.zero?
|
35
|
-
log! "
|
40
|
+
log! "Timed out after #{interval} seconds"
|
36
41
|
procline paused? ? "Paused" : "Waiting for #{@queues.join(',')}"
|
37
|
-
sleep interval
|
38
42
|
end
|
39
43
|
end
|
40
44
|
|
41
45
|
ensure
|
42
46
|
unregister_worker
|
47
|
+
wait_for_shutdown if locked?
|
43
48
|
end
|
44
49
|
|
45
|
-
def
|
46
|
-
redis.
|
47
|
-
sleep 1 while scaling? and not shutdown?
|
48
|
-
redis.del("scale:#{self}")
|
50
|
+
def should_lock?
|
51
|
+
redis.exists(:lock)
|
49
52
|
end
|
50
53
|
|
51
|
-
def
|
52
|
-
redis.
|
53
|
-
|
54
|
+
def lock
|
55
|
+
redis.sadd(:locked, self)
|
56
|
+
@locked = true
|
54
57
|
end
|
55
58
|
|
56
|
-
def
|
57
|
-
|
59
|
+
def locked?
|
60
|
+
@locked
|
58
61
|
end
|
59
62
|
|
60
|
-
def
|
61
|
-
|
62
|
-
|
63
|
+
def should_unlock?
|
64
|
+
return false if should_lock?
|
65
|
+
locked?
|
66
|
+
end
|
63
67
|
|
64
|
-
|
68
|
+
def wait_for_shutdown
|
69
|
+
sleep 0.1 until shutdown? or should_unlock?
|
70
|
+
end
|
65
71
|
|
66
|
-
|
72
|
+
def self.locked
|
73
|
+
Array(redis.smembers(:locked))
|
74
|
+
end
|
67
75
|
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
end
|
72
|
-
rescue Redis::Distributed::CannotDistribute
|
73
|
-
names.each do |name|
|
74
|
-
value = redis.get name
|
75
|
-
reportedly_scaling[name] = value unless value.nil? || value.empty?
|
76
|
-
end
|
77
|
-
end
|
76
|
+
def self.lock
|
77
|
+
redis.set(:lock, true)
|
78
|
+
end
|
78
79
|
|
79
|
-
|
80
|
-
|
81
|
-
|
80
|
+
def self.unlock
|
81
|
+
redis.del(:lock)
|
82
|
+
redis.del(:locked)
|
82
83
|
end
|
83
|
-
end
|
84
84
|
|
85
|
-
|
85
|
+
def self.prune
|
86
|
+
all_workers = Worker.all
|
87
|
+
all_workers.each do |worker|
|
88
|
+
worker.unregister_worker
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
end
|
93
|
+
end
|
data/test/config_test.rb
CHANGED
@@ -10,7 +10,7 @@ class ConfigTest < MiniTest::Unit::TestCase
|
|
10
10
|
end
|
11
11
|
|
12
12
|
def test_scale_interval_default
|
13
|
-
assert_equal
|
13
|
+
assert_equal 5, @config.scale_interval
|
14
14
|
end
|
15
15
|
|
16
16
|
def test_poll_interval_default
|
@@ -44,7 +44,7 @@ class ConfigTest < MiniTest::Unit::TestCase
|
|
44
44
|
def test_scale_interval
|
45
45
|
@config.scale_interval = 99
|
46
46
|
assert_equal 99, @config.scale_interval
|
47
|
-
@config.scale_interval =
|
47
|
+
@config.scale_interval = 5
|
48
48
|
end
|
49
49
|
|
50
50
|
def test_poll_interval
|
data/test/heroku_scaler_test.rb
CHANGED
@@ -6,8 +6,8 @@ class HerokuScalerTest < MiniTest::Unit::TestCase
|
|
6
6
|
@config = Resque::Plugins::HerokuScaler::Config
|
7
7
|
|
8
8
|
@redis = mock('Mock Redis')
|
9
|
-
@redis.stubs(:set).with(:
|
10
|
-
@redis.stubs(:del).with(:
|
9
|
+
@redis.stubs(:set).with(:lock, true).returns(true)
|
10
|
+
@redis.stubs(:del).with(:lock).returns(true)
|
11
11
|
Resque.stubs(:redis).returns(@redis)
|
12
12
|
|
13
13
|
@manager = mock('Mock Manager')
|
@@ -15,14 +15,14 @@ class HerokuScalerTest < MiniTest::Unit::TestCase
|
|
15
15
|
end
|
16
16
|
|
17
17
|
def test_no_scale_for_zero_jobs
|
18
|
-
Resque.stubs(:info).returns({ :pending => 0, :
|
18
|
+
Resque.stubs(:info).returns({ :pending => 0, :locked => 0 })
|
19
19
|
@manager.expects(:workers).returns(0)
|
20
20
|
@manager.expects(:workers=).never
|
21
21
|
@scaler.scale()
|
22
22
|
end
|
23
23
|
|
24
24
|
def test_scale_up_for_pending_job
|
25
|
-
Resque.stubs(:info).returns({ :pending => 1, :
|
25
|
+
Resque.stubs(:info).returns({ :pending => 1, :locked => 0 })
|
26
26
|
@manager.expects(:workers).returns(0)
|
27
27
|
@manager.expects(:workers=).with(1)
|
28
28
|
@scaler.scale()
|
@@ -30,17 +30,25 @@ class HerokuScalerTest < MiniTest::Unit::TestCase
|
|
30
30
|
|
31
31
|
def test_scale_down_timeout
|
32
32
|
@config.scale_timeout = 1
|
33
|
-
Resque.stubs(:info).returns({ :pending => 0, :
|
33
|
+
Resque.stubs(:info).returns({ :pending => 0, :locked => 1 })
|
34
34
|
@manager.expects(:workers).returns(1)
|
35
35
|
@manager.expects(:workers=).with(0)
|
36
|
+
@scaler.expects(:offline?).returns(false)
|
37
|
+
@scaler.expects(:prune).never
|
38
|
+
@scaler.expects(:lock).returns(true)
|
39
|
+
@scaler.expects(:unlock).returns(true)
|
36
40
|
@scaler.scale()
|
37
41
|
@config.scale_timeout = 90
|
38
42
|
end
|
39
43
|
|
40
44
|
def test_scale_down_for_zero_jobs
|
41
|
-
Resque.stubs(:info).returns({ :pending => 0, :
|
45
|
+
Resque.stubs(:info).returns({ :pending => 0, :locked => 1 })
|
42
46
|
@manager.expects(:workers).returns(1)
|
43
47
|
@manager.expects(:workers=).with(0)
|
48
|
+
@scaler.expects(:offline?).returns(true)
|
49
|
+
@scaler.expects(:prune).returns(true)
|
50
|
+
@scaler.expects(:lock).returns(true)
|
51
|
+
@scaler.expects(:unlock).returns(true)
|
44
52
|
@scaler.scale()
|
45
53
|
end
|
46
54
|
|
data/test/worker_test.rb
CHANGED
@@ -1,26 +1,15 @@
|
|
1
1
|
require 'test_helper'
|
2
2
|
|
3
3
|
class WorkerTest < MiniTest::Unit::TestCase
|
4
|
-
def
|
5
|
-
Resque.redis.stubs(:exists).with(:
|
6
|
-
Resque.redis.expects(:
|
7
|
-
Resque.redis.expects(:del).with(regexp_matches(/^scale:(.)*$/))
|
4
|
+
def test_lock
|
5
|
+
Resque.redis.stubs(:exists).with(:lock).returns(true)
|
6
|
+
Resque.redis.expects(:sadd).with(:locked, kind_of(Resque::Worker))
|
8
7
|
|
9
|
-
worker = Resque::Worker.new(['*'])
|
10
|
-
worker.stubs(:shutdown?).returns(true)
|
8
|
+
worker = Resque::Worker.new(['*'])
|
11
9
|
worker.stubs(:register_worker).returns(true)
|
12
10
|
worker.stubs(:unregister_worker).returns(true)
|
13
|
-
worker.
|
14
|
-
|
15
|
-
|
16
|
-
def test_unregister_worker
|
17
|
-
Resque.redis.expects(:del).with(regexp_matches(/^scale:(.)*$/))
|
18
|
-
Resque.redis.stubs(:del).with(regexp_matches(/^worker:(.)*$/))
|
19
|
-
Resque.redis.stubs(:del).with(regexp_matches(/^stat:(.)*$/))
|
20
|
-
|
21
|
-
worker = Resque::Worker.new(['*'])
|
22
|
-
worker.stubs(:shutdown?).returns(true)
|
23
|
-
worker.stubs(:register_worker).returns(true)
|
11
|
+
worker.stubs(:shutdown?).returns(false)
|
12
|
+
worker.expects(:wait_for_shutdown).returns(true)
|
24
13
|
worker.work(0)
|
25
14
|
end
|
26
15
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: resque-heroku-scaler
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,30 +9,30 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-
|
12
|
+
date: 2012-06-28 00:00:00.000000000Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: resque
|
16
|
-
requirement: &
|
16
|
+
requirement: &70351775059460 !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
19
|
- - ~>
|
20
20
|
- !ruby/object:Gem::Version
|
21
|
-
version: 1.
|
21
|
+
version: 1.20.0
|
22
22
|
type: :runtime
|
23
23
|
prerelease: false
|
24
|
-
version_requirements: *
|
24
|
+
version_requirements: *70351775059460
|
25
25
|
- !ruby/object:Gem::Dependency
|
26
26
|
name: heroku
|
27
|
-
requirement: &
|
27
|
+
requirement: &70351775058060 !ruby/object:Gem::Requirement
|
28
28
|
none: false
|
29
29
|
requirements:
|
30
30
|
- - ~>
|
31
31
|
- !ruby/object:Gem::Version
|
32
|
-
version: 2.
|
32
|
+
version: 2.28.7
|
33
33
|
type: :runtime
|
34
34
|
prerelease: false
|
35
|
-
version_requirements: *
|
35
|
+
version_requirements: *70351775058060
|
36
36
|
description: ! ' This gem provides autoscaling for Resque workers on Heroku.
|
37
37
|
|
38
38
|
'
|