erchef-expander 11.4.0
Sign up to get free protection for your applications and to get access to all the features.
- data/LICENSE +201 -0
- data/README.rdoc +0 -0
- data/bin/chef-expander +29 -0
- data/bin/chef-expander-vnode +30 -0
- data/bin/chef-expanderctl +30 -0
- data/conf/chef-expander.rb.example +9 -0
- data/lib/chef/expander.rb +36 -0
- data/lib/chef/expander/cluster_supervisor.rb +130 -0
- data/lib/chef/expander/configuration.rb +320 -0
- data/lib/chef/expander/control.rb +206 -0
- data/lib/chef/expander/daemonizable.rb +150 -0
- data/lib/chef/expander/flattener.rb +79 -0
- data/lib/chef/expander/loggable.rb +40 -0
- data/lib/chef/expander/logger.rb +135 -0
- data/lib/chef/expander/node.rb +177 -0
- data/lib/chef/expander/solrizer.rb +275 -0
- data/lib/chef/expander/version.rb +41 -0
- data/lib/chef/expander/vnode.rb +106 -0
- data/lib/chef/expander/vnode_supervisor.rb +265 -0
- data/lib/chef/expander/vnode_table.rb +83 -0
- data/scripts/check_queue_size +93 -0
- data/scripts/check_queue_size_munin +51 -0
- data/scripts/make_solr_xml +58 -0
- data/scripts/traffic-creator +97 -0
- metadata +284 -0
@@ -0,0 +1,320 @@
|
|
1
|
+
#
|
2
|
+
# Author:: Daniel DeLeo (<dan@opscode.com>)
|
3
|
+
# Author:: Seth Falcon (<seth@opscode.com>)
|
4
|
+
# Author:: Chris Walters (<cw@opscode.com>)
|
5
|
+
# Copyright:: Copyright (c) 2010-2011 Opscode, Inc.
|
6
|
+
# License:: Apache License, Version 2.0
|
7
|
+
#
|
8
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
9
|
+
# you may not use this file except in compliance with the License.
|
10
|
+
# You may obtain a copy of the License at
|
11
|
+
#
|
12
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
13
|
+
#
|
14
|
+
# Unless required by applicable law or agreed to in writing, software
|
15
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
16
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
17
|
+
# See the License for the specific language governing permissions and
|
18
|
+
# limitations under the License.
|
19
|
+
#
|
20
|
+
|
21
|
+
require 'pp'
|
22
|
+
require 'optparse'
|
23
|
+
require 'singleton'
|
24
|
+
|
25
|
+
require 'chef/expander/flattener'
|
26
|
+
require 'chef/expander/loggable'
|
27
|
+
require 'chef/expander/version'
|
28
|
+
|
29
|
+
module Chef
|
30
|
+
module Expander
|
31
|
+
|
32
|
+
def self.config
|
33
|
+
@config ||= Configuration::Base.new
|
34
|
+
end
|
35
|
+
|
36
|
+
def self.init_config(argv)
|
37
|
+
config.apply_defaults
|
38
|
+
remaining_opts_after_parse = Configuration::CLI.parse_options(argv)
|
39
|
+
# Need to be able to override the default config file location on the command line
|
40
|
+
config_file_to_use = Configuration::CLI.config.config_file || config.config_file
|
41
|
+
config.merge_config(Configuration::Base.from_chef_compat_config(config_file_to_use))
|
42
|
+
# But for all other config options, the CLI config should win over config file
|
43
|
+
config.merge_config(Configuration::CLI.config)
|
44
|
+
config.validate!
|
45
|
+
remaining_opts_after_parse
|
46
|
+
end
|
47
|
+
|
48
|
+
class ChefCompatibleConfig
|
49
|
+
|
50
|
+
attr_reader :config_hash
|
51
|
+
|
52
|
+
def initialize
|
53
|
+
@config_hash = {}
|
54
|
+
end
|
55
|
+
|
56
|
+
def load(file)
|
57
|
+
file = File.expand_path(file)
|
58
|
+
instance_eval(IO.read(file), file, 1) if File.readable?(file)
|
59
|
+
end
|
60
|
+
|
61
|
+
def method_missing(method_name, *args, &block)
|
62
|
+
if args.size == 1
|
63
|
+
@config_hash[method_name] = args.first
|
64
|
+
elsif args.empty?
|
65
|
+
@config_hash[method_name] or super
|
66
|
+
else
|
67
|
+
super
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
end
|
72
|
+
|
73
|
+
module Configuration
|
74
|
+
|
75
|
+
class InvalidConfiguration < StandardError
|
76
|
+
end
|
77
|
+
|
78
|
+
class Base
|
79
|
+
|
80
|
+
DEFAULT_PIDFILE = Object.new
|
81
|
+
|
82
|
+
include Loggable
|
83
|
+
|
84
|
+
def self.from_chef_compat_config(file)
|
85
|
+
config = ChefCompatibleConfig.new
|
86
|
+
config.load(file)
|
87
|
+
from_hash(config.config_hash)
|
88
|
+
end
|
89
|
+
|
90
|
+
def self.from_hash(config_hash)
|
91
|
+
config = new
|
92
|
+
config_hash.each do |setting, value|
|
93
|
+
setter = "#{setting}=".to_sym
|
94
|
+
if config.respond_to?(setter)
|
95
|
+
config.send(setter, value)
|
96
|
+
end
|
97
|
+
end
|
98
|
+
config
|
99
|
+
end
|
100
|
+
|
101
|
+
def self.configurables
|
102
|
+
@configurables ||= []
|
103
|
+
end
|
104
|
+
|
105
|
+
def self.validations
|
106
|
+
@validations ||= []
|
107
|
+
end
|
108
|
+
|
109
|
+
def self.defaults
|
110
|
+
@defaults ||= {}
|
111
|
+
end
|
112
|
+
|
113
|
+
def self.configurable(setting, default=nil, &validation)
|
114
|
+
attr_accessor(setting)
|
115
|
+
configurables << setting
|
116
|
+
defaults[setting] = default
|
117
|
+
validations << validation if block_given?
|
118
|
+
|
119
|
+
setting
|
120
|
+
end
|
121
|
+
|
122
|
+
configurable :config_file, "/etc/chef/solr.rb" do
|
123
|
+
unless (config_file && File.exist?(config_file) && File.readable?(config_file))
|
124
|
+
log.warn {"* " * 40}
|
125
|
+
log.warn {"Config file #{config_file} does not exist or cannot be read by user (#{Process.euid})"}
|
126
|
+
log.warn {"Default configuration settings will be used"}
|
127
|
+
log.warn {"* " * 40}
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
131
|
+
configurable :index do
|
132
|
+
unless index.nil? # in single-cluster mode, this setting is not required.
|
133
|
+
invalid("You must specify this node's position in the ring as an integer") unless index.kind_of?(Integer)
|
134
|
+
invalid("The index cannot be larger than the cluster size (node-count)") unless (index.to_i <= node_count.to_i)
|
135
|
+
end
|
136
|
+
end
|
137
|
+
|
138
|
+
configurable :node_count, 1 do
|
139
|
+
invalid("You must specify the node_count as an integer") unless node_count.kind_of?(Integer)
|
140
|
+
invalid("The node_count must be 1 or greater") unless node_count >= 1
|
141
|
+
invalid("The node_count cannot be smaller than the index") unless node_count >= index.to_i
|
142
|
+
end
|
143
|
+
|
144
|
+
configurable :ps_tag, ""
|
145
|
+
|
146
|
+
configurable :solr_url, "http://localhost:8983/solr"
|
147
|
+
|
148
|
+
# override the setter for solr_url for backward compatibilty
|
149
|
+
def solr_url=(url)
|
150
|
+
if url && url == "http://localhost:8983"
|
151
|
+
log.warn {"You seem to have a legacy setting for solr_url: did you mean #{url}/solr ?"}
|
152
|
+
url = "#{url}/solr"
|
153
|
+
end
|
154
|
+
@solr_url = url
|
155
|
+
end
|
156
|
+
|
157
|
+
configurable :amqp_host, '0.0.0.0'
|
158
|
+
|
159
|
+
configurable :amqp_port, 5672
|
160
|
+
|
161
|
+
configurable :amqp_user, 'chef'
|
162
|
+
|
163
|
+
configurable :amqp_pass, 'testing'
|
164
|
+
|
165
|
+
configurable :amqp_vhost, '/chef'
|
166
|
+
|
167
|
+
configurable :user, nil
|
168
|
+
|
169
|
+
configurable :group, nil
|
170
|
+
|
171
|
+
configurable :daemonize, false
|
172
|
+
|
173
|
+
alias :daemonize? :daemonize
|
174
|
+
|
175
|
+
configurable :pidfile, DEFAULT_PIDFILE
|
176
|
+
|
177
|
+
def pidfile
|
178
|
+
if @pidfile.equal?(DEFAULT_PIDFILE)
|
179
|
+
Process.euid == 0 ? '/var/run/chef-expander.pid' : '/tmp/chef-expander.pid'
|
180
|
+
else
|
181
|
+
@pidfile
|
182
|
+
end
|
183
|
+
end
|
184
|
+
|
185
|
+
configurable :log_level, :info
|
186
|
+
|
187
|
+
# override the setter for log_level to also actually set the level
|
188
|
+
def log_level=(level)
|
189
|
+
if level #don't accept nil for an answer
|
190
|
+
level = level.to_sym
|
191
|
+
Loggable::LOGGER.level = level
|
192
|
+
@log_level = log_level
|
193
|
+
end
|
194
|
+
level
|
195
|
+
end
|
196
|
+
|
197
|
+
configurable :log_location, STDOUT
|
198
|
+
|
199
|
+
# override the setter for log_location to re-init the logger
|
200
|
+
def log_location=(location)
|
201
|
+
Loggable::LOGGER.init(location) unless location.nil?
|
202
|
+
end
|
203
|
+
|
204
|
+
def initialize
|
205
|
+
reset!
|
206
|
+
end
|
207
|
+
|
208
|
+
def reset!(stdout=nil)
|
209
|
+
self.class.configurables.each do |setting|
|
210
|
+
send("#{setting}=".to_sym, nil)
|
211
|
+
end
|
212
|
+
@stdout = stdout || STDOUT
|
213
|
+
end
|
214
|
+
|
215
|
+
def apply_defaults
|
216
|
+
self.class.defaults.each do |setting, value|
|
217
|
+
self.send("#{setting}=".to_sym, value)
|
218
|
+
end
|
219
|
+
end
|
220
|
+
|
221
|
+
def merge_config(other)
|
222
|
+
self.class.configurables.each do |setting|
|
223
|
+
value = other.send(setting)
|
224
|
+
self.send("#{setting}=".to_sym, value) if value
|
225
|
+
end
|
226
|
+
end
|
227
|
+
|
228
|
+
def fail_if_invalid
|
229
|
+
validate!
|
230
|
+
rescue InvalidConfiguration => e
|
231
|
+
@stdout.puts("Invalid configuration: #{e.message}")
|
232
|
+
exit(1)
|
233
|
+
end
|
234
|
+
|
235
|
+
def invalid(message)
|
236
|
+
raise InvalidConfiguration, message
|
237
|
+
end
|
238
|
+
|
239
|
+
def validate!
|
240
|
+
self.class.validations.each do |validation_proc|
|
241
|
+
instance_eval(&validation_proc)
|
242
|
+
end
|
243
|
+
end
|
244
|
+
|
245
|
+
def vnode_numbers
|
246
|
+
vnodes_per_node = VNODES / node_count
|
247
|
+
lower_bound = (index - 1) * vnodes_per_node
|
248
|
+
upper_bound = lower_bound + vnodes_per_node
|
249
|
+
upper_bound += VNODES % vnodes_per_node if index == node_count
|
250
|
+
(lower_bound...upper_bound).to_a
|
251
|
+
end
|
252
|
+
|
253
|
+
def amqp_config
|
254
|
+
{:host => amqp_host, :port => amqp_port, :user => amqp_user, :pass => amqp_pass, :vhost => amqp_vhost}
|
255
|
+
end
|
256
|
+
|
257
|
+
end
|
258
|
+
|
259
|
+
module CLI
|
260
|
+
@config = Configuration::Base.new
|
261
|
+
|
262
|
+
@option_parser = OptionParser.new do |o|
|
263
|
+
o.banner = "Usage: chef-expander [options]"
|
264
|
+
|
265
|
+
o.on('-c', '--config CONFIG_FILE', 'a configuration file to use') do |conf|
|
266
|
+
@config.config_file = File.expand_path(conf)
|
267
|
+
end
|
268
|
+
|
269
|
+
o.on('-i', '--index INDEX', 'the slot this node will occupy in the ring') do |i|
|
270
|
+
@config.index = i.to_i
|
271
|
+
end
|
272
|
+
|
273
|
+
o.on('-n', '--node-count NUMBER', 'the number of nodes in the ring') do |n|
|
274
|
+
@config.node_count = n.to_i
|
275
|
+
end
|
276
|
+
|
277
|
+
o.on('-l', '--log-level LOG_LEVEL', 'set the log level') do |l|
|
278
|
+
@config.log_level = l
|
279
|
+
end
|
280
|
+
|
281
|
+
o.on('-L', '--logfile LOG_LOCATION', 'Logfile to use') do |l|
|
282
|
+
@config.log_location = l
|
283
|
+
end
|
284
|
+
|
285
|
+
o.on('-d', '--daemonize', 'fork into the background') do
|
286
|
+
@config.daemonize = true
|
287
|
+
end
|
288
|
+
|
289
|
+
o.on('-P', '--pid PIDFILE') do |p|
|
290
|
+
@config.pidfile = p
|
291
|
+
end
|
292
|
+
|
293
|
+
o.on_tail('-h', '--help', 'show this message') do
|
294
|
+
puts "chef-expander #{Expander.version}"
|
295
|
+
puts ''
|
296
|
+
puts o
|
297
|
+
exit 1
|
298
|
+
end
|
299
|
+
|
300
|
+
o.on_tail('-v', '--version', 'show the version and exit') do
|
301
|
+
puts "chef-expander #{Expander.version}"
|
302
|
+
exit 0
|
303
|
+
end
|
304
|
+
|
305
|
+
end
|
306
|
+
|
307
|
+
def self.parse_options(argv)
|
308
|
+
@option_parser.parse!(argv.dup)
|
309
|
+
end
|
310
|
+
|
311
|
+
def self.config
|
312
|
+
@config
|
313
|
+
end
|
314
|
+
|
315
|
+
end
|
316
|
+
|
317
|
+
end
|
318
|
+
|
319
|
+
end
|
320
|
+
end
|
@@ -0,0 +1,206 @@
|
|
1
|
+
#
|
2
|
+
# Author:: Daniel DeLeo (<dan@opscode.com>)
|
3
|
+
# Author:: Seth Falcon (<seth@opscode.com>)
|
4
|
+
# Author:: Chris Walters (<cw@opscode.com>)
|
5
|
+
# Copyright:: Copyright (c) 2010-2011 Opscode, Inc.
|
6
|
+
# License:: Apache License, Version 2.0
|
7
|
+
#
|
8
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
9
|
+
# you may not use this file except in compliance with the License.
|
10
|
+
# You may obtain a copy of the License at
|
11
|
+
#
|
12
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
13
|
+
#
|
14
|
+
# Unless required by applicable law or agreed to in writing, software
|
15
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
16
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
17
|
+
# See the License for the specific language governing permissions and
|
18
|
+
# limitations under the License.
|
19
|
+
#
|
20
|
+
|
21
|
+
require 'bunny'
|
22
|
+
require 'yajl'
|
23
|
+
require 'eventmachine'
|
24
|
+
require 'amqp'
|
25
|
+
require 'mq'
|
26
|
+
require 'highline'
|
27
|
+
|
28
|
+
require 'chef/expander/node'
|
29
|
+
require 'chef/expander/configuration'
|
30
|
+
|
31
|
+
require 'pp'
|
32
|
+
|
33
|
+
module Chef
|
34
|
+
module Expander
|
35
|
+
class Control
|
36
|
+
|
37
|
+
def self.run(argv)
|
38
|
+
remaining_args_after_opts = Expander.init_config(ARGV)
|
39
|
+
new(remaining_args_after_opts).run
|
40
|
+
end
|
41
|
+
|
42
|
+
def self.desc(description)
|
43
|
+
@desc = description
|
44
|
+
end
|
45
|
+
|
46
|
+
def self.option(*args)
|
47
|
+
#TODO
|
48
|
+
end
|
49
|
+
|
50
|
+
def self.arg(*args)
|
51
|
+
#TODO
|
52
|
+
end
|
53
|
+
|
54
|
+
def self.descriptions
|
55
|
+
@descriptions ||= []
|
56
|
+
end
|
57
|
+
|
58
|
+
def self.method_added(method_name)
|
59
|
+
if @desc
|
60
|
+
descriptions << [method_name, method_name.to_s.gsub('_', '-'), @desc]
|
61
|
+
@desc = nil
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
#--
|
66
|
+
# TODO: this is confusing and unneeded. Just whitelist the methods
|
67
|
+
# that map to commands and use +send+
|
68
|
+
def self.compile
|
69
|
+
run_method = "def run; case @argv.first;"
|
70
|
+
descriptions.each do |method_name, command_name, desc|
|
71
|
+
run_method << "when '#{command_name}';#{method_name};"
|
72
|
+
end
|
73
|
+
run_method << "else; help; end; end;"
|
74
|
+
class_eval(run_method, __FILE__, __LINE__)
|
75
|
+
end
|
76
|
+
|
77
|
+
def initialize(argv)
|
78
|
+
@argv = argv.dup
|
79
|
+
end
|
80
|
+
|
81
|
+
desc "Show this message"
|
82
|
+
def help
|
83
|
+
puts "Chef Expander #{Expander.version}"
|
84
|
+
puts "Usage: chef-expanderctl COMMAND"
|
85
|
+
puts
|
86
|
+
puts "Commands:"
|
87
|
+
self.class.descriptions.each do |method_name, command_name, desc|
|
88
|
+
puts " #{command_name}".ljust(15) + desc
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
desc "display the aggregate queue backlog"
|
93
|
+
def queue_depth
|
94
|
+
h = HighLine.new
|
95
|
+
message_counts = []
|
96
|
+
|
97
|
+
amqp_client = Bunny.new(Expander.config.amqp_config)
|
98
|
+
amqp_client.start
|
99
|
+
|
100
|
+
0.upto(VNODES - 1) do |vnode|
|
101
|
+
q = amqp_client.queue("vnode-#{vnode}", :durable => true)
|
102
|
+
message_counts << q.status[:message_count]
|
103
|
+
end
|
104
|
+
total_messages = message_counts.inject(0) { |sum, count| sum + count }
|
105
|
+
max = message_counts.max
|
106
|
+
min = message_counts.min
|
107
|
+
|
108
|
+
avg = total_messages.to_f / message_counts.size.to_f
|
109
|
+
|
110
|
+
puts " total messages: #{total_messages}"
|
111
|
+
puts " average queue depth: #{avg}"
|
112
|
+
puts " max queue depth: #{max}"
|
113
|
+
puts " min queue depth: #{min}"
|
114
|
+
ensure
|
115
|
+
amqp_client.stop if defined?(amqp_client) && amqp_client
|
116
|
+
end
|
117
|
+
|
118
|
+
desc "show the backlog and consumer count for each vnode queue"
|
119
|
+
def queue_status
|
120
|
+
h = HighLine.new
|
121
|
+
queue_status = [h.color("VNode", :bold), h.color("Messages", :bold), h.color("Consumers", :bold)]
|
122
|
+
|
123
|
+
total_messages = 0
|
124
|
+
|
125
|
+
amqp_client = Bunny.new(Expander.config.amqp_config)
|
126
|
+
amqp_client.start
|
127
|
+
|
128
|
+
0.upto(VNODES - 1) do |vnode|
|
129
|
+
q = amqp_client.queue("vnode-#{vnode}", :durable => true)
|
130
|
+
status = q.status
|
131
|
+
# returns {:message_count => method.message_count, :consumer_count => method.consumer_count}
|
132
|
+
queue_status << vnode.to_s << status[:message_count].to_s << status[:consumer_count].to_s
|
133
|
+
total_messages += status[:message_count]
|
134
|
+
end
|
135
|
+
puts " total messages: #{total_messages}"
|
136
|
+
puts
|
137
|
+
puts h.list(queue_status, :columns_across, 3)
|
138
|
+
ensure
|
139
|
+
amqp_client.stop if defined?(amqp_client) && amqp_client
|
140
|
+
end
|
141
|
+
|
142
|
+
desc "show the status of the nodes in the cluster"
|
143
|
+
def node_status
|
144
|
+
status_mutex = Mutex.new
|
145
|
+
h = ::HighLine.new
|
146
|
+
node_status = [h.color("Host", :bold), h.color("PID", :bold), h.color("GUID", :bold), h.color("Vnodes", :bold)]
|
147
|
+
|
148
|
+
print("Collecting status info from the cluster...")
|
149
|
+
|
150
|
+
AMQP.start(Expander.config.amqp_config) do
|
151
|
+
node = Expander::Node.local_node
|
152
|
+
node.exclusive_control_queue.subscribe do |header, message|
|
153
|
+
status = Yajl::Parser.parse(message)
|
154
|
+
status_mutex.synchronize do
|
155
|
+
node_status << status["hostname_f"]
|
156
|
+
node_status << status["pid"].to_s
|
157
|
+
node_status << status["guid"]
|
158
|
+
# BIG ASSUMPTION HERE that nodes only have contiguous vnode ranges
|
159
|
+
# will not be true once vnode recovery is implemented
|
160
|
+
node_status << "#{status["vnodes"].min}-#{status["vnodes"].max}"
|
161
|
+
end
|
162
|
+
end
|
163
|
+
node.broadcast_message(Yajl::Encoder.encode(:action => :status, :rsvp => node.exclusive_control_queue_name))
|
164
|
+
EM.add_timer(2) { AMQP.stop;EM.stop }
|
165
|
+
end
|
166
|
+
|
167
|
+
puts "done"
|
168
|
+
puts
|
169
|
+
puts h.list(node_status, :columns_across, 4)
|
170
|
+
puts
|
171
|
+
end
|
172
|
+
|
173
|
+
desc "sets the log level of all nodes in the cluster"
|
174
|
+
def log_level
|
175
|
+
@argv.shift
|
176
|
+
level = @argv.first
|
177
|
+
acceptable_levels = %w{debug info warn error fatal}
|
178
|
+
unless acceptable_levels.include?(level)
|
179
|
+
puts "Log level must be one of #{acceptable_levels.join(', ')}"
|
180
|
+
exit 1
|
181
|
+
end
|
182
|
+
|
183
|
+
h = HighLine.new
|
184
|
+
response_mutex = Mutex.new
|
185
|
+
|
186
|
+
responses = [h.color("Host", :bold), h.color("PID", :bold), h.color("GUID", :bold), h.color("Log Level", :bold)]
|
187
|
+
AMQP.start(Expander.config.amqp_config) do
|
188
|
+
node = Expander::Node.local_node
|
189
|
+
node.exclusive_control_queue.subscribe do |header, message|
|
190
|
+
reply = Yajl::Parser.parse(message)
|
191
|
+
n = reply['node']
|
192
|
+
response_mutex.synchronize do
|
193
|
+
responses << n["hostname_f"] << n["pid"].to_s << n["guid"] << reply["level"]
|
194
|
+
end
|
195
|
+
end
|
196
|
+
node.broadcast_message(Yajl::Encoder.encode({:action => :set_log_level, :level => level, :rsvp => node.exclusive_control_queue_name}))
|
197
|
+
EM.add_timer(2) { AMQP.stop; EM.stop }
|
198
|
+
end
|
199
|
+
puts h.list(responses, :columns_across, 4)
|
200
|
+
end
|
201
|
+
|
202
|
+
|
203
|
+
compile
|
204
|
+
end
|
205
|
+
end
|
206
|
+
end
|