resque-cluster 0.1.1.1 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/Gemfile +3 -1
- data/Gemfile.lock +47 -44
- data/lib/resque/cluster.rb +1 -0
- data/lib/resque/cluster/config.rb +169 -0
- data/lib/resque/cluster/config/file.rb +40 -0
- data/lib/resque/cluster/config/verifier.rb +27 -0
- data/lib/resque/cluster/member.rb +17 -27
- data/lib/resque/cluster/version.rb +1 -1
- data/lib/resque/pool/patches.rb +5 -0
- data/resque-cluster.gemspec +1 -1
- data/spec/integration/cluster_spec.rb +41 -11
- data/spec/integration/config/bad_global_config.yml +3 -0
- data/spec/integration/config/global_rebalance_config3.yml +5 -0
- data/spec/integration/config/rebalance_config.yml +4 -0
- data/spec/integration/spec_helper.rb +3 -0
- data/spec/integration/test_member_manager.rb +50 -19
- data/spec/unit/config/verifier_spec.rb +86 -0
- data/spec/unit/config_spec.rb +194 -0
- data/spec/unit/member_spec.rb +5 -32
- data/spec/unit/resque_pool_patches_spec.rb +3 -3
- data/spec/unit/spec_helper.rb +5 -1
- data/spec/unit/support/broken_yaml_config.yml +1 -0
- data/spec/unit/support/empty_config.yml +1 -0
- data/spec/unit/support/missing_global_maximum.yml +3 -0
- data/spec/unit/support/missing_local_maximum.yml +3 -0
- data/spec/unit/support/non_hash_config.yml +0 -0
- data/spec/unit/support/separate_missing_global.yml +2 -0
- data/spec/unit/support/separate_missing_local.yml +2 -0
- data/spec/unit/support/single_old_missing_global.yml +9 -0
- data/spec/unit/support/single_old_missing_local.yml +9 -0
- data/spec/unit/support/valid_config.yml +13 -0
- data/spec/unit/support/valid_global_config.yml +1 -0
- data/spec/unit/support/valid_local_config.yml +1 -0
- data/spec/unit/support/valid_single_new_config.yml +13 -0
- data/spec/unit/support/valid_single_old_config.yml +10 -0
- metadata +72 -30
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: ff333888f274415257e79dbb38df4cea39f85aec
|
4
|
+
data.tar.gz: 2e4633a132a0b8ca17e96ac13e37446cb88c06e0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 60b4c0e0c44a770c927ae9395b63b28774ea5bc728f527ba9d612ae14f498671335cd7e288918c2898f0a01d33d83a266e72706bbc6145860f6b4c86c474b2c1
|
7
|
+
data.tar.gz: 6ba4a2699620059ffd384fffba71eabf3e44a1c2778636934787ea9f422b6b85d79f74d225938fa422ca67ef0ad666c63bff40764ef2933005a86082a0508ca2
|
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,32 +1,29 @@
|
|
1
1
|
GEM
|
2
2
|
remote: http://rubygems.org/
|
3
3
|
specs:
|
4
|
-
addressable (2.
|
5
|
-
ast (2.
|
6
|
-
|
7
|
-
parser (~> 2.2)
|
8
|
-
awesome_print (1.6.1)
|
4
|
+
addressable (2.4.0)
|
5
|
+
ast (2.3.0)
|
6
|
+
awesome_print (1.7.0)
|
9
7
|
builder (3.2.2)
|
10
|
-
coderay (1.1.
|
8
|
+
coderay (1.1.1)
|
11
9
|
descendants_tracker (0.0.4)
|
12
10
|
thread_safe (~> 0.3, >= 0.3.1)
|
13
11
|
diff-lcs (1.2.5)
|
14
12
|
docile (1.1.5)
|
15
|
-
faraday (0.9.
|
13
|
+
faraday (0.9.2)
|
16
14
|
multipart-post (>= 1.2, < 3)
|
17
|
-
|
18
|
-
|
19
|
-
|
15
|
+
ffi (1.9.14)
|
16
|
+
git (1.3.0)
|
17
|
+
github_api (0.14.5)
|
18
|
+
addressable (~> 2.4.0)
|
20
19
|
descendants_tracker (~> 0.0.4)
|
21
20
|
faraday (~> 0.8, < 0.10)
|
22
|
-
hashie (>= 3.
|
23
|
-
|
24
|
-
|
25
|
-
oauth2
|
26
|
-
gru (0.1.2)
|
21
|
+
hashie (>= 3.4)
|
22
|
+
oauth2 (~> 1.0)
|
23
|
+
gru (0.1.3)
|
27
24
|
redis (> 0.0)
|
28
|
-
hashie (3.4.
|
29
|
-
highline (1.7.
|
25
|
+
hashie (3.4.4)
|
26
|
+
highline (1.7.8)
|
30
27
|
jeweler (2.0.1)
|
31
28
|
builder
|
32
29
|
bundler (>= 1.0)
|
@@ -37,40 +34,42 @@ GEM
|
|
37
34
|
rake
|
38
35
|
rdoc
|
39
36
|
json (1.8.3)
|
40
|
-
jwt (1.5.
|
37
|
+
jwt (1.5.4)
|
41
38
|
method_source (0.8.2)
|
42
|
-
|
43
|
-
mock_redis (0.15.
|
39
|
+
mini_portile2 (2.1.0)
|
40
|
+
mock_redis (0.15.4)
|
44
41
|
mono_logger (1.1.0)
|
45
|
-
multi_json (1.
|
42
|
+
multi_json (1.12.1)
|
46
43
|
multi_xml (0.5.5)
|
47
44
|
multipart-post (2.0.0)
|
48
|
-
nokogiri (1.6.
|
49
|
-
|
50
|
-
|
45
|
+
nokogiri (1.6.8)
|
46
|
+
mini_portile2 (~> 2.1.0)
|
47
|
+
pkg-config (~> 1.1.7)
|
48
|
+
oauth2 (1.2.0)
|
51
49
|
faraday (>= 0.8, < 0.10)
|
52
50
|
jwt (~> 1.0)
|
53
51
|
multi_json (~> 1.3)
|
54
52
|
multi_xml (~> 0.5)
|
55
|
-
rack (
|
56
|
-
parser (2.
|
57
|
-
ast (
|
53
|
+
rack (>= 1.2, < 3)
|
54
|
+
parser (2.3.1.2)
|
55
|
+
ast (~> 2.2)
|
56
|
+
pkg-config (1.1.7)
|
58
57
|
powerpack (0.1.1)
|
59
|
-
pry (0.10.
|
58
|
+
pry (0.10.4)
|
60
59
|
coderay (~> 1.1.0)
|
61
60
|
method_source (~> 0.8.1)
|
62
61
|
slop (~> 3.4)
|
63
62
|
rack (1.6.4)
|
64
63
|
rack-protection (1.5.3)
|
65
64
|
rack
|
66
|
-
rainbow (2.
|
67
|
-
rake (
|
65
|
+
rainbow (2.1.0)
|
66
|
+
rake (11.2.2)
|
68
67
|
rdoc (3.12.2)
|
69
68
|
json (~> 1.4)
|
70
|
-
redis (3.
|
69
|
+
redis (3.3.1)
|
71
70
|
redis-namespace (1.5.2)
|
72
71
|
redis (~> 3.0, >= 3.0.4)
|
73
|
-
resque (1.
|
72
|
+
resque (1.26.0)
|
74
73
|
mono_logger (~> 1.0)
|
75
74
|
multi_json (~> 1.0)
|
76
75
|
redis-namespace (~> 1.3)
|
@@ -92,26 +91,28 @@ GEM
|
|
92
91
|
rspec-mocks (3.1.3)
|
93
92
|
rspec-support (~> 3.1.0)
|
94
93
|
rspec-support (3.1.2)
|
95
|
-
rubocop (0.
|
96
|
-
|
97
|
-
parser (>= 2.2.2.5, < 3.0)
|
94
|
+
rubocop (0.42.0)
|
95
|
+
parser (>= 2.3.1.1, < 3.0)
|
98
96
|
powerpack (~> 0.1)
|
99
97
|
rainbow (>= 1.99.1, < 3.0)
|
100
|
-
ruby-progressbar (~> 1.
|
101
|
-
|
102
|
-
|
98
|
+
ruby-progressbar (~> 1.7)
|
99
|
+
unicode-display_width (~> 1.0, >= 1.0.1)
|
100
|
+
ruby-progressbar (1.8.1)
|
101
|
+
simplecov (0.12.0)
|
103
102
|
docile (~> 1.1.0)
|
104
|
-
json (
|
103
|
+
json (>= 1.8, < 3)
|
105
104
|
simplecov-html (~> 0.10.0)
|
106
105
|
simplecov-html (0.10.0)
|
107
|
-
sinatra (1.4.
|
108
|
-
rack (~> 1.
|
106
|
+
sinatra (1.4.7)
|
107
|
+
rack (~> 1.5)
|
109
108
|
rack-protection (~> 1.4)
|
110
109
|
tilt (>= 1.3, < 3)
|
111
110
|
slop (3.6.0)
|
111
|
+
sys-proctable (1.1.1)
|
112
112
|
thread_safe (0.3.5)
|
113
|
-
tilt (2.0.
|
113
|
+
tilt (2.0.5)
|
114
114
|
trollop (2.1.2)
|
115
|
+
unicode-display_width (1.1.0)
|
115
116
|
vegas (0.1.11)
|
116
117
|
rack (>= 1.0.0)
|
117
118
|
|
@@ -121,7 +122,8 @@ PLATFORMS
|
|
121
122
|
DEPENDENCIES
|
122
123
|
awesome_print
|
123
124
|
bundler (~> 1.0)
|
124
|
-
|
125
|
+
ffi
|
126
|
+
gru
|
125
127
|
jeweler (~> 2.0.1)
|
126
128
|
mock_redis (~> 0.15.0)
|
127
129
|
pry
|
@@ -130,6 +132,7 @@ DEPENDENCIES
|
|
130
132
|
rspec (~> 3.1.0)
|
131
133
|
rubocop (~> 0.31)
|
132
134
|
simplecov
|
135
|
+
sys-proctable
|
133
136
|
|
134
137
|
BUNDLED WITH
|
135
|
-
1.
|
138
|
+
1.12.5
|
data/lib/resque/cluster.rb
CHANGED
@@ -0,0 +1,169 @@
|
|
1
|
+
require 'resque/cluster/config/file'
|
2
|
+
require 'resque/cluster/config/verifier'
|
3
|
+
|
4
|
+
module Resque
|
5
|
+
class Cluster
|
6
|
+
# Config is a global and local configuration of a member of a resque pool cluster
|
7
|
+
class Config
|
8
|
+
extend Forwardable
|
9
|
+
|
10
|
+
attr_reader :configs, :config, :global_config, :verifier, :version_git_hash
|
11
|
+
|
12
|
+
def initialize(config_path, global_config_path = nil)
|
13
|
+
@config = Config::File.new(config_path)
|
14
|
+
|
15
|
+
@configs = [config]
|
16
|
+
|
17
|
+
if global_config_path
|
18
|
+
global_config = Config::File.new(global_config_path)
|
19
|
+
|
20
|
+
if global_config.expand_path != config.expand_path
|
21
|
+
@global_config = global_config
|
22
|
+
|
23
|
+
@configs << global_config
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
@errors = Set.new
|
28
|
+
@verifier = Verifier.new(configs)
|
29
|
+
@version_git_hash = config_version
|
30
|
+
end
|
31
|
+
|
32
|
+
def verified?
|
33
|
+
verifier.verified? && complete_worker_config?
|
34
|
+
end
|
35
|
+
|
36
|
+
def gru_format
|
37
|
+
return {} unless verified?
|
38
|
+
|
39
|
+
{
|
40
|
+
manage_worker_heartbeats: true,
|
41
|
+
host_maximums: host_maximums,
|
42
|
+
client_settings: Resque.redis.client.options,
|
43
|
+
cluster_name: Cluster.config[:cluster_name],
|
44
|
+
environment_name: Cluster.config[:environment],
|
45
|
+
cluster_maximums: cluster_maximums,
|
46
|
+
rebalance_flag: rebalance_flag || false,
|
47
|
+
max_workers_per_host: max_workers_per_host || nil,
|
48
|
+
presume_host_dead_after: presume_dead_after || 120,
|
49
|
+
version_hash: version_git_hash
|
50
|
+
}
|
51
|
+
end
|
52
|
+
|
53
|
+
def errors
|
54
|
+
@errors + configs.map { |config| config.errors.map { |error| "#{config}: #{error}" } }.flatten
|
55
|
+
end
|
56
|
+
|
57
|
+
def warnings
|
58
|
+
@warnings ||= []
|
59
|
+
end
|
60
|
+
|
61
|
+
def log_warnings
|
62
|
+
warnings.each do |warning|
|
63
|
+
puts warning
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
def log_errors
|
68
|
+
errors.each do |error|
|
69
|
+
puts error
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
private
|
74
|
+
|
75
|
+
def complete_worker_config?
|
76
|
+
host = host_maximums.delete_if { |_, v| v.nil? }
|
77
|
+
cluster = cluster_maximums.delete_if { |_, v| v.nil? }
|
78
|
+
|
79
|
+
(host.keys == cluster.keys).tap do |complete|
|
80
|
+
@errors << "Every worker configuration must contain a local and a global maximum." unless complete
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
def host_maximums
|
85
|
+
case config_type
|
86
|
+
when :separate
|
87
|
+
config.contents
|
88
|
+
when :old
|
89
|
+
config.contents.reject do |k, _|
|
90
|
+
['global_maximums', 'presume_dead_after', 'max_workers_per_host', 'rebalance_cluster'].include?(k)
|
91
|
+
end
|
92
|
+
when :new
|
93
|
+
config['workers'].each.with_object({}) do |(pool, maximums), local_maximums|
|
94
|
+
local_maximums[pool] = maximums['local']
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
def cluster_maximums
|
100
|
+
case config_type
|
101
|
+
when :separate
|
102
|
+
global_config['global_maximums'] || global_config.contents.reject do |k, _|
|
103
|
+
['global_maximums', 'presume_dead_after', 'max_workers_per_host', 'rebalance_cluster'].include?(k)
|
104
|
+
end
|
105
|
+
when :old
|
106
|
+
config['global_maximums']
|
107
|
+
when :new
|
108
|
+
config['workers'].each.with_object({}) do |(pool, maximums), global_maximums|
|
109
|
+
global_maximums[pool] = maximums['global']
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
def rebalance_flag
|
115
|
+
case config_type
|
116
|
+
when :separate
|
117
|
+
global_config['rebalance_cluster']
|
118
|
+
when :old, :new
|
119
|
+
config['rebalance_cluster']
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
123
|
+
def max_workers_per_host
|
124
|
+
case config_type
|
125
|
+
when :separate
|
126
|
+
global_config['max_workers_per_host']
|
127
|
+
when :old, :new
|
128
|
+
config['max_workers_per_host']
|
129
|
+
end
|
130
|
+
end
|
131
|
+
|
132
|
+
def presume_dead_after
|
133
|
+
case config_type
|
134
|
+
when :separate
|
135
|
+
global_config['presume_dead_after']
|
136
|
+
when :old, :new
|
137
|
+
config['presume_dead_after']
|
138
|
+
end
|
139
|
+
end
|
140
|
+
|
141
|
+
def config_type
|
142
|
+
@config_type ||=
|
143
|
+
if global_config
|
144
|
+
:separate
|
145
|
+
elsif config['workers']
|
146
|
+
:new
|
147
|
+
else
|
148
|
+
:old
|
149
|
+
end
|
150
|
+
end
|
151
|
+
|
152
|
+
def config_version
|
153
|
+
return unless verified?
|
154
|
+
|
155
|
+
directory_name = config.dirname
|
156
|
+
|
157
|
+
if directory_name.exist?
|
158
|
+
output = Dir.chdir(directory_name) { `git rev-parse --verify HEAD`.chomp }
|
159
|
+
|
160
|
+
if $?.success?
|
161
|
+
@version_git_hash = output
|
162
|
+
else
|
163
|
+
@warnings << "Your config directory: #{directory_name} is not a git repo. Your configuration will not be versioned"
|
164
|
+
end
|
165
|
+
end
|
166
|
+
end
|
167
|
+
end
|
168
|
+
end
|
169
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
module Resque
|
2
|
+
class Cluster
|
3
|
+
class Config
|
4
|
+
class File < Pathname
|
5
|
+
extend Forwardable
|
6
|
+
|
7
|
+
delegate :[] => :contents
|
8
|
+
|
9
|
+
def exist?
|
10
|
+
super.tap { |exists| errors << "Configuration file doesn't exist" unless exists }
|
11
|
+
end
|
12
|
+
|
13
|
+
def valid?
|
14
|
+
if contents.is_a?(Hash)
|
15
|
+
errors << "Config file is empty" unless contents.any?
|
16
|
+
else
|
17
|
+
errors << "Parsed config as invalid type: expected Hash, got #{contents.class}"
|
18
|
+
end
|
19
|
+
|
20
|
+
contents.is_a?(Hash) && contents.any?
|
21
|
+
end
|
22
|
+
|
23
|
+
def errors
|
24
|
+
@errors ||= Set.new
|
25
|
+
end
|
26
|
+
|
27
|
+
def contents
|
28
|
+
@contents ||=
|
29
|
+
begin
|
30
|
+
YAML.load(ERB.new(self.read).result)
|
31
|
+
rescue => e
|
32
|
+
errors << e.message
|
33
|
+
|
34
|
+
nil
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
module Resque
|
2
|
+
class Cluster
|
3
|
+
class Config
|
4
|
+
class Verifier
|
5
|
+
attr_reader :configs
|
6
|
+
|
7
|
+
def initialize(configs)
|
8
|
+
@configs = configs
|
9
|
+
end
|
10
|
+
|
11
|
+
def verified?
|
12
|
+
configs_exist? && configs_are_valid?
|
13
|
+
end
|
14
|
+
|
15
|
+
private
|
16
|
+
|
17
|
+
def configs_exist?
|
18
|
+
configs.all?(&:exist?)
|
19
|
+
end
|
20
|
+
|
21
|
+
def configs_are_valid?
|
22
|
+
configs.all?(&:valid?)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -5,14 +5,18 @@ module Resque
|
|
5
5
|
class Cluster
|
6
6
|
# Member is a single member of a resque pool cluster
|
7
7
|
class Member
|
8
|
-
attr_reader :hostname, :pool, :
|
8
|
+
attr_reader :hostname, :pool, :config
|
9
9
|
|
10
10
|
def initialize(started_pool)
|
11
11
|
@pool = started_pool
|
12
|
-
@
|
13
|
-
@
|
14
|
-
|
15
|
-
|
12
|
+
@config = Config.new(Cluster.config[:local_config_path], Cluster.config[:global_config_path])
|
13
|
+
if @config.verified?
|
14
|
+
@config.log_warnings
|
15
|
+
@worker_count_manager = initialize_gru
|
16
|
+
else
|
17
|
+
@config.log_errors
|
18
|
+
@pool.quit
|
19
|
+
end
|
16
20
|
end
|
17
21
|
|
18
22
|
def perform
|
@@ -25,6 +29,7 @@ module Resque
|
|
25
29
|
end
|
26
30
|
|
27
31
|
def check_for_worker_count_adjustment
|
32
|
+
return unless gru_is_inititalized?
|
28
33
|
host_count_adjustment = @worker_count_manager.adjust_workers
|
29
34
|
adjust_worker_counts(host_count_adjustment) if host_count_adjustment
|
30
35
|
end
|
@@ -32,7 +37,7 @@ module Resque
|
|
32
37
|
private
|
33
38
|
|
34
39
|
def global_prefix
|
35
|
-
"cluster:#{Cluster.config[:cluster_name]}:#{Cluster.config[:environment]}"
|
40
|
+
"cluster:#{Cluster.config[:cluster_name]}:#{Cluster.config[:environment]}:#{@config.version_git_hash}"
|
36
41
|
end
|
37
42
|
|
38
43
|
def member_prefix
|
@@ -44,7 +49,7 @@ module Resque
|
|
44
49
|
end
|
45
50
|
|
46
51
|
def initialize_gru
|
47
|
-
Gru.create(
|
52
|
+
Gru.create(@config.gru_format)
|
48
53
|
end
|
49
54
|
|
50
55
|
def hostname
|
@@ -53,29 +58,23 @@ module Resque
|
|
53
58
|
|
54
59
|
def adjust_worker_counts(count_adjustments)
|
55
60
|
count_adjustments.each do |worker, count|
|
56
|
-
next if count == 0
|
57
61
|
@pool.adjust_worker_counts(worker, count)
|
58
62
|
update_counts
|
59
63
|
end
|
60
64
|
end
|
61
65
|
|
62
|
-
def parse_config(config_path)
|
63
|
-
return {} unless config_path && File.exist?(config_path)
|
64
|
-
YAML.load(ERB.new(IO.read(config_path)).result)
|
65
|
-
end
|
66
|
-
|
67
66
|
def remove_counts
|
68
67
|
Resque.redis.del(running_workers_key_name)
|
69
68
|
end
|
70
69
|
|
71
70
|
def unqueue_all_workers
|
72
|
-
@worker_count_manager.release_workers
|
71
|
+
@worker_count_manager.release_workers if gru_is_inititalized?
|
73
72
|
end
|
74
73
|
|
75
74
|
def unqueue_workers(workers)
|
76
75
|
workers = Array(workers)
|
77
76
|
workers.each do |worker|
|
78
|
-
@worker_count_manager.release_workers(worker)
|
77
|
+
@worker_count_manager.release_workers(worker) if gru_is_inititalized?
|
79
78
|
end
|
80
79
|
end
|
81
80
|
|
@@ -86,19 +85,10 @@ module Resque
|
|
86
85
|
end
|
87
86
|
end
|
88
87
|
|
89
|
-
def
|
90
|
-
|
91
|
-
cluster_maximums: @global_config["global_maximums"] || @global_config,
|
92
|
-
host_maximums: @local_config,
|
93
|
-
client_settings: Resque.redis.client.options,
|
94
|
-
rebalance_flag: @global_config["rebalance_cluster"] || false,
|
95
|
-
max_workers_per_host: @global_config["max_workers_per_host"] || nil,
|
96
|
-
cluster_name: Cluster.config[:cluster_name],
|
97
|
-
environment_name: Cluster.config[:environment],
|
98
|
-
presume_host_dead_after: @global_config["presume_dead_after"] || 120,
|
99
|
-
manage_worker_heartbeats: true
|
100
|
-
}
|
88
|
+
def gru_is_inititalized?
|
89
|
+
! @worker_count_manager.nil?
|
101
90
|
end
|
91
|
+
|
102
92
|
end
|
103
93
|
end
|
104
94
|
end
|