erchef-expander 11.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,320 @@
1
+ #
2
+ # Author:: Daniel DeLeo (<dan@opscode.com>)
3
+ # Author:: Seth Falcon (<seth@opscode.com>)
4
+ # Author:: Chris Walters (<cw@opscode.com>)
5
+ # Copyright:: Copyright (c) 2010-2011 Opscode, Inc.
6
+ # License:: Apache License, Version 2.0
7
+ #
8
+ # Licensed under the Apache License, Version 2.0 (the "License");
9
+ # you may not use this file except in compliance with the License.
10
+ # You may obtain a copy of the License at
11
+ #
12
+ # http://www.apache.org/licenses/LICENSE-2.0
13
+ #
14
+ # Unless required by applicable law or agreed to in writing, software
15
+ # distributed under the License is distributed on an "AS IS" BASIS,
16
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17
+ # See the License for the specific language governing permissions and
18
+ # limitations under the License.
19
+ #
20
+
21
+ require 'pp'
22
+ require 'optparse'
23
+ require 'singleton'
24
+
25
+ require 'chef/expander/flattener'
26
+ require 'chef/expander/loggable'
27
+ require 'chef/expander/version'
28
+
29
+ module Chef
30
+ module Expander
31
+
32
+ def self.config
33
+ @config ||= Configuration::Base.new
34
+ end
35
+
36
+ def self.init_config(argv)
37
+ config.apply_defaults
38
+ remaining_opts_after_parse = Configuration::CLI.parse_options(argv)
39
+ # Need to be able to override the default config file location on the command line
40
+ config_file_to_use = Configuration::CLI.config.config_file || config.config_file
41
+ config.merge_config(Configuration::Base.from_chef_compat_config(config_file_to_use))
42
+ # But for all other config options, the CLI config should win over config file
43
+ config.merge_config(Configuration::CLI.config)
44
+ config.validate!
45
+ remaining_opts_after_parse
46
+ end
47
+
48
+ class ChefCompatibleConfig
49
+
50
+ attr_reader :config_hash
51
+
52
+ def initialize
53
+ @config_hash = {}
54
+ end
55
+
56
+ def load(file)
57
+ file = File.expand_path(file)
58
+ instance_eval(IO.read(file), file, 1) if File.readable?(file)
59
+ end
60
+
61
+ def method_missing(method_name, *args, &block)
62
+ if args.size == 1
63
+ @config_hash[method_name] = args.first
64
+ elsif args.empty?
65
+ @config_hash[method_name] or super
66
+ else
67
+ super
68
+ end
69
+ end
70
+
71
+ end
72
+
73
+ module Configuration
74
+
75
+ class InvalidConfiguration < StandardError
76
+ end
77
+
78
+ class Base
79
+
80
+ DEFAULT_PIDFILE = Object.new
81
+
82
+ include Loggable
83
+
84
+ def self.from_chef_compat_config(file)
85
+ config = ChefCompatibleConfig.new
86
+ config.load(file)
87
+ from_hash(config.config_hash)
88
+ end
89
+
90
+ def self.from_hash(config_hash)
91
+ config = new
92
+ config_hash.each do |setting, value|
93
+ setter = "#{setting}=".to_sym
94
+ if config.respond_to?(setter)
95
+ config.send(setter, value)
96
+ end
97
+ end
98
+ config
99
+ end
100
+
101
+ def self.configurables
102
+ @configurables ||= []
103
+ end
104
+
105
+ def self.validations
106
+ @validations ||= []
107
+ end
108
+
109
+ def self.defaults
110
+ @defaults ||= {}
111
+ end
112
+
113
+ def self.configurable(setting, default=nil, &validation)
114
+ attr_accessor(setting)
115
+ configurables << setting
116
+ defaults[setting] = default
117
+ validations << validation if block_given?
118
+
119
+ setting
120
+ end
121
+
122
+ configurable :config_file, "/etc/chef/solr.rb" do
123
+ unless (config_file && File.exist?(config_file) && File.readable?(config_file))
124
+ log.warn {"* " * 40}
125
+ log.warn {"Config file #{config_file} does not exist or cannot be read by user (#{Process.euid})"}
126
+ log.warn {"Default configuration settings will be used"}
127
+ log.warn {"* " * 40}
128
+ end
129
+ end
130
+
131
+ configurable :index do
132
+ unless index.nil? # in single-cluster mode, this setting is not required.
133
+ invalid("You must specify this node's position in the ring as an integer") unless index.kind_of?(Integer)
134
+ invalid("The index cannot be larger than the cluster size (node-count)") unless (index.to_i <= node_count.to_i)
135
+ end
136
+ end
137
+
138
+ configurable :node_count, 1 do
139
+ invalid("You must specify the node_count as an integer") unless node_count.kind_of?(Integer)
140
+ invalid("The node_count must be 1 or greater") unless node_count >= 1
141
+ invalid("The node_count cannot be smaller than the index") unless node_count >= index.to_i
142
+ end
143
+
144
+ configurable :ps_tag, ""
145
+
146
+ configurable :solr_url, "http://localhost:8983/solr"
147
+
148
+ # override the setter for solr_url for backward compatibilty
149
+ def solr_url=(url)
150
+ if url && url == "http://localhost:8983"
151
+ log.warn {"You seem to have a legacy setting for solr_url: did you mean #{url}/solr ?"}
152
+ url = "#{url}/solr"
153
+ end
154
+ @solr_url = url
155
+ end
156
+
157
+ configurable :amqp_host, '0.0.0.0'
158
+
159
+ configurable :amqp_port, 5672
160
+
161
+ configurable :amqp_user, 'chef'
162
+
163
+ configurable :amqp_pass, 'testing'
164
+
165
+ configurable :amqp_vhost, '/chef'
166
+
167
+ configurable :user, nil
168
+
169
+ configurable :group, nil
170
+
171
+ configurable :daemonize, false
172
+
173
+ alias :daemonize? :daemonize
174
+
175
+ configurable :pidfile, DEFAULT_PIDFILE
176
+
177
+ def pidfile
178
+ if @pidfile.equal?(DEFAULT_PIDFILE)
179
+ Process.euid == 0 ? '/var/run/chef-expander.pid' : '/tmp/chef-expander.pid'
180
+ else
181
+ @pidfile
182
+ end
183
+ end
184
+
185
+ configurable :log_level, :info
186
+
187
+ # override the setter for log_level to also actually set the level
188
+ def log_level=(level)
189
+ if level #don't accept nil for an answer
190
+ level = level.to_sym
191
+ Loggable::LOGGER.level = level
192
+ @log_level = log_level
193
+ end
194
+ level
195
+ end
196
+
197
+ configurable :log_location, STDOUT
198
+
199
+ # override the setter for log_location to re-init the logger
200
+ def log_location=(location)
201
+ Loggable::LOGGER.init(location) unless location.nil?
202
+ end
203
+
204
+ def initialize
205
+ reset!
206
+ end
207
+
208
+ def reset!(stdout=nil)
209
+ self.class.configurables.each do |setting|
210
+ send("#{setting}=".to_sym, nil)
211
+ end
212
+ @stdout = stdout || STDOUT
213
+ end
214
+
215
+ def apply_defaults
216
+ self.class.defaults.each do |setting, value|
217
+ self.send("#{setting}=".to_sym, value)
218
+ end
219
+ end
220
+
221
+ def merge_config(other)
222
+ self.class.configurables.each do |setting|
223
+ value = other.send(setting)
224
+ self.send("#{setting}=".to_sym, value) if value
225
+ end
226
+ end
227
+
228
+ def fail_if_invalid
229
+ validate!
230
+ rescue InvalidConfiguration => e
231
+ @stdout.puts("Invalid configuration: #{e.message}")
232
+ exit(1)
233
+ end
234
+
235
+ def invalid(message)
236
+ raise InvalidConfiguration, message
237
+ end
238
+
239
+ def validate!
240
+ self.class.validations.each do |validation_proc|
241
+ instance_eval(&validation_proc)
242
+ end
243
+ end
244
+
245
+ def vnode_numbers
246
+ vnodes_per_node = VNODES / node_count
247
+ lower_bound = (index - 1) * vnodes_per_node
248
+ upper_bound = lower_bound + vnodes_per_node
249
+ upper_bound += VNODES % vnodes_per_node if index == node_count
250
+ (lower_bound...upper_bound).to_a
251
+ end
252
+
253
+ def amqp_config
254
+ {:host => amqp_host, :port => amqp_port, :user => amqp_user, :pass => amqp_pass, :vhost => amqp_vhost}
255
+ end
256
+
257
+ end
258
+
259
+ module CLI
260
+ @config = Configuration::Base.new
261
+
262
+ @option_parser = OptionParser.new do |o|
263
+ o.banner = "Usage: chef-expander [options]"
264
+
265
+ o.on('-c', '--config CONFIG_FILE', 'a configuration file to use') do |conf|
266
+ @config.config_file = File.expand_path(conf)
267
+ end
268
+
269
+ o.on('-i', '--index INDEX', 'the slot this node will occupy in the ring') do |i|
270
+ @config.index = i.to_i
271
+ end
272
+
273
+ o.on('-n', '--node-count NUMBER', 'the number of nodes in the ring') do |n|
274
+ @config.node_count = n.to_i
275
+ end
276
+
277
+ o.on('-l', '--log-level LOG_LEVEL', 'set the log level') do |l|
278
+ @config.log_level = l
279
+ end
280
+
281
+ o.on('-L', '--logfile LOG_LOCATION', 'Logfile to use') do |l|
282
+ @config.log_location = l
283
+ end
284
+
285
+ o.on('-d', '--daemonize', 'fork into the background') do
286
+ @config.daemonize = true
287
+ end
288
+
289
+ o.on('-P', '--pid PIDFILE') do |p|
290
+ @config.pidfile = p
291
+ end
292
+
293
+ o.on_tail('-h', '--help', 'show this message') do
294
+ puts "chef-expander #{Expander.version}"
295
+ puts ''
296
+ puts o
297
+ exit 1
298
+ end
299
+
300
+ o.on_tail('-v', '--version', 'show the version and exit') do
301
+ puts "chef-expander #{Expander.version}"
302
+ exit 0
303
+ end
304
+
305
+ end
306
+
307
+ def self.parse_options(argv)
308
+ @option_parser.parse!(argv.dup)
309
+ end
310
+
311
+ def self.config
312
+ @config
313
+ end
314
+
315
+ end
316
+
317
+ end
318
+
319
+ end
320
+ end
@@ -0,0 +1,206 @@
1
+ #
2
+ # Author:: Daniel DeLeo (<dan@opscode.com>)
3
+ # Author:: Seth Falcon (<seth@opscode.com>)
4
+ # Author:: Chris Walters (<cw@opscode.com>)
5
+ # Copyright:: Copyright (c) 2010-2011 Opscode, Inc.
6
+ # License:: Apache License, Version 2.0
7
+ #
8
+ # Licensed under the Apache License, Version 2.0 (the "License");
9
+ # you may not use this file except in compliance with the License.
10
+ # You may obtain a copy of the License at
11
+ #
12
+ # http://www.apache.org/licenses/LICENSE-2.0
13
+ #
14
+ # Unless required by applicable law or agreed to in writing, software
15
+ # distributed under the License is distributed on an "AS IS" BASIS,
16
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17
+ # See the License for the specific language governing permissions and
18
+ # limitations under the License.
19
+ #
20
+
21
+ require 'bunny'
22
+ require 'yajl'
23
+ require 'eventmachine'
24
+ require 'amqp'
25
+ require 'mq'
26
+ require 'highline'
27
+
28
+ require 'chef/expander/node'
29
+ require 'chef/expander/configuration'
30
+
31
+ require 'pp'
32
+
33
+ module Chef
34
+ module Expander
35
+ class Control
36
+
37
+ def self.run(argv)
38
+ remaining_args_after_opts = Expander.init_config(ARGV)
39
+ new(remaining_args_after_opts).run
40
+ end
41
+
42
+ def self.desc(description)
43
+ @desc = description
44
+ end
45
+
46
+ def self.option(*args)
47
+ #TODO
48
+ end
49
+
50
+ def self.arg(*args)
51
+ #TODO
52
+ end
53
+
54
+ def self.descriptions
55
+ @descriptions ||= []
56
+ end
57
+
58
+ def self.method_added(method_name)
59
+ if @desc
60
+ descriptions << [method_name, method_name.to_s.gsub('_', '-'), @desc]
61
+ @desc = nil
62
+ end
63
+ end
64
+
65
+ #--
66
+ # TODO: this is confusing and unneeded. Just whitelist the methods
67
+ # that map to commands and use +send+
68
+ def self.compile
69
+ run_method = "def run; case @argv.first;"
70
+ descriptions.each do |method_name, command_name, desc|
71
+ run_method << "when '#{command_name}';#{method_name};"
72
+ end
73
+ run_method << "else; help; end; end;"
74
+ class_eval(run_method, __FILE__, __LINE__)
75
+ end
76
+
77
+ def initialize(argv)
78
+ @argv = argv.dup
79
+ end
80
+
81
+ desc "Show this message"
82
+ def help
83
+ puts "Chef Expander #{Expander.version}"
84
+ puts "Usage: chef-expanderctl COMMAND"
85
+ puts
86
+ puts "Commands:"
87
+ self.class.descriptions.each do |method_name, command_name, desc|
88
+ puts " #{command_name}".ljust(15) + desc
89
+ end
90
+ end
91
+
92
+ desc "display the aggregate queue backlog"
93
+ def queue_depth
94
+ h = HighLine.new
95
+ message_counts = []
96
+
97
+ amqp_client = Bunny.new(Expander.config.amqp_config)
98
+ amqp_client.start
99
+
100
+ 0.upto(VNODES - 1) do |vnode|
101
+ q = amqp_client.queue("vnode-#{vnode}", :durable => true)
102
+ message_counts << q.status[:message_count]
103
+ end
104
+ total_messages = message_counts.inject(0) { |sum, count| sum + count }
105
+ max = message_counts.max
106
+ min = message_counts.min
107
+
108
+ avg = total_messages.to_f / message_counts.size.to_f
109
+
110
+ puts " total messages: #{total_messages}"
111
+ puts " average queue depth: #{avg}"
112
+ puts " max queue depth: #{max}"
113
+ puts " min queue depth: #{min}"
114
+ ensure
115
+ amqp_client.stop if defined?(amqp_client) && amqp_client
116
+ end
117
+
118
+ desc "show the backlog and consumer count for each vnode queue"
119
+ def queue_status
120
+ h = HighLine.new
121
+ queue_status = [h.color("VNode", :bold), h.color("Messages", :bold), h.color("Consumers", :bold)]
122
+
123
+ total_messages = 0
124
+
125
+ amqp_client = Bunny.new(Expander.config.amqp_config)
126
+ amqp_client.start
127
+
128
+ 0.upto(VNODES - 1) do |vnode|
129
+ q = amqp_client.queue("vnode-#{vnode}", :durable => true)
130
+ status = q.status
131
+ # returns {:message_count => method.message_count, :consumer_count => method.consumer_count}
132
+ queue_status << vnode.to_s << status[:message_count].to_s << status[:consumer_count].to_s
133
+ total_messages += status[:message_count]
134
+ end
135
+ puts " total messages: #{total_messages}"
136
+ puts
137
+ puts h.list(queue_status, :columns_across, 3)
138
+ ensure
139
+ amqp_client.stop if defined?(amqp_client) && amqp_client
140
+ end
141
+
142
+ desc "show the status of the nodes in the cluster"
143
+ def node_status
144
+ status_mutex = Mutex.new
145
+ h = ::HighLine.new
146
+ node_status = [h.color("Host", :bold), h.color("PID", :bold), h.color("GUID", :bold), h.color("Vnodes", :bold)]
147
+
148
+ print("Collecting status info from the cluster...")
149
+
150
+ AMQP.start(Expander.config.amqp_config) do
151
+ node = Expander::Node.local_node
152
+ node.exclusive_control_queue.subscribe do |header, message|
153
+ status = Yajl::Parser.parse(message)
154
+ status_mutex.synchronize do
155
+ node_status << status["hostname_f"]
156
+ node_status << status["pid"].to_s
157
+ node_status << status["guid"]
158
+ # BIG ASSUMPTION HERE that nodes only have contiguous vnode ranges
159
+ # will not be true once vnode recovery is implemented
160
+ node_status << "#{status["vnodes"].min}-#{status["vnodes"].max}"
161
+ end
162
+ end
163
+ node.broadcast_message(Yajl::Encoder.encode(:action => :status, :rsvp => node.exclusive_control_queue_name))
164
+ EM.add_timer(2) { AMQP.stop;EM.stop }
165
+ end
166
+
167
+ puts "done"
168
+ puts
169
+ puts h.list(node_status, :columns_across, 4)
170
+ puts
171
+ end
172
+
173
+ desc "sets the log level of all nodes in the cluster"
174
+ def log_level
175
+ @argv.shift
176
+ level = @argv.first
177
+ acceptable_levels = %w{debug info warn error fatal}
178
+ unless acceptable_levels.include?(level)
179
+ puts "Log level must be one of #{acceptable_levels.join(', ')}"
180
+ exit 1
181
+ end
182
+
183
+ h = HighLine.new
184
+ response_mutex = Mutex.new
185
+
186
+ responses = [h.color("Host", :bold), h.color("PID", :bold), h.color("GUID", :bold), h.color("Log Level", :bold)]
187
+ AMQP.start(Expander.config.amqp_config) do
188
+ node = Expander::Node.local_node
189
+ node.exclusive_control_queue.subscribe do |header, message|
190
+ reply = Yajl::Parser.parse(message)
191
+ n = reply['node']
192
+ response_mutex.synchronize do
193
+ responses << n["hostname_f"] << n["pid"].to_s << n["guid"] << reply["level"]
194
+ end
195
+ end
196
+ node.broadcast_message(Yajl::Encoder.encode({:action => :set_log_level, :level => level, :rsvp => node.exclusive_control_queue_name}))
197
+ EM.add_timer(2) { AMQP.stop; EM.stop }
198
+ end
199
+ puts h.list(responses, :columns_across, 4)
200
+ end
201
+
202
+
203
+ compile
204
+ end
205
+ end
206
+ end