erchef-expander 11.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/LICENSE +201 -0
- data/README.rdoc +0 -0
- data/bin/chef-expander +29 -0
- data/bin/chef-expander-vnode +30 -0
- data/bin/chef-expanderctl +30 -0
- data/conf/chef-expander.rb.example +9 -0
- data/lib/chef/expander.rb +36 -0
- data/lib/chef/expander/cluster_supervisor.rb +130 -0
- data/lib/chef/expander/configuration.rb +320 -0
- data/lib/chef/expander/control.rb +206 -0
- data/lib/chef/expander/daemonizable.rb +150 -0
- data/lib/chef/expander/flattener.rb +79 -0
- data/lib/chef/expander/loggable.rb +40 -0
- data/lib/chef/expander/logger.rb +135 -0
- data/lib/chef/expander/node.rb +177 -0
- data/lib/chef/expander/solrizer.rb +275 -0
- data/lib/chef/expander/version.rb +41 -0
- data/lib/chef/expander/vnode.rb +106 -0
- data/lib/chef/expander/vnode_supervisor.rb +265 -0
- data/lib/chef/expander/vnode_table.rb +83 -0
- data/scripts/check_queue_size +93 -0
- data/scripts/check_queue_size_munin +51 -0
- data/scripts/make_solr_xml +58 -0
- data/scripts/traffic-creator +97 -0
- metadata +284 -0
@@ -0,0 +1,320 @@
|
|
1
|
+
#
|
2
|
+
# Author:: Daniel DeLeo (<dan@opscode.com>)
|
3
|
+
# Author:: Seth Falcon (<seth@opscode.com>)
|
4
|
+
# Author:: Chris Walters (<cw@opscode.com>)
|
5
|
+
# Copyright:: Copyright (c) 2010-2011 Opscode, Inc.
|
6
|
+
# License:: Apache License, Version 2.0
|
7
|
+
#
|
8
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
9
|
+
# you may not use this file except in compliance with the License.
|
10
|
+
# You may obtain a copy of the License at
|
11
|
+
#
|
12
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
13
|
+
#
|
14
|
+
# Unless required by applicable law or agreed to in writing, software
|
15
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
16
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
17
|
+
# See the License for the specific language governing permissions and
|
18
|
+
# limitations under the License.
|
19
|
+
#
|
20
|
+
|
21
|
+
require 'pp'
|
22
|
+
require 'optparse'
|
23
|
+
require 'singleton'
|
24
|
+
|
25
|
+
require 'chef/expander/flattener'
|
26
|
+
require 'chef/expander/loggable'
|
27
|
+
require 'chef/expander/version'
|
28
|
+
|
29
|
+
module Chef
|
30
|
+
module Expander
|
31
|
+
|
32
|
+
def self.config
|
33
|
+
@config ||= Configuration::Base.new
|
34
|
+
end
|
35
|
+
|
36
|
+
def self.init_config(argv)
|
37
|
+
config.apply_defaults
|
38
|
+
remaining_opts_after_parse = Configuration::CLI.parse_options(argv)
|
39
|
+
# Need to be able to override the default config file location on the command line
|
40
|
+
config_file_to_use = Configuration::CLI.config.config_file || config.config_file
|
41
|
+
config.merge_config(Configuration::Base.from_chef_compat_config(config_file_to_use))
|
42
|
+
# But for all other config options, the CLI config should win over config file
|
43
|
+
config.merge_config(Configuration::CLI.config)
|
44
|
+
config.validate!
|
45
|
+
remaining_opts_after_parse
|
46
|
+
end
|
47
|
+
|
48
|
+
class ChefCompatibleConfig
|
49
|
+
|
50
|
+
attr_reader :config_hash
|
51
|
+
|
52
|
+
def initialize
|
53
|
+
@config_hash = {}
|
54
|
+
end
|
55
|
+
|
56
|
+
def load(file)
|
57
|
+
file = File.expand_path(file)
|
58
|
+
instance_eval(IO.read(file), file, 1) if File.readable?(file)
|
59
|
+
end
|
60
|
+
|
61
|
+
def method_missing(method_name, *args, &block)
|
62
|
+
if args.size == 1
|
63
|
+
@config_hash[method_name] = args.first
|
64
|
+
elsif args.empty?
|
65
|
+
@config_hash[method_name] or super
|
66
|
+
else
|
67
|
+
super
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
end
|
72
|
+
|
73
|
+
module Configuration
|
74
|
+
|
75
|
+
class InvalidConfiguration < StandardError
|
76
|
+
end
|
77
|
+
|
78
|
+
class Base
|
79
|
+
|
80
|
+
DEFAULT_PIDFILE = Object.new
|
81
|
+
|
82
|
+
include Loggable
|
83
|
+
|
84
|
+
def self.from_chef_compat_config(file)
|
85
|
+
config = ChefCompatibleConfig.new
|
86
|
+
config.load(file)
|
87
|
+
from_hash(config.config_hash)
|
88
|
+
end
|
89
|
+
|
90
|
+
def self.from_hash(config_hash)
|
91
|
+
config = new
|
92
|
+
config_hash.each do |setting, value|
|
93
|
+
setter = "#{setting}=".to_sym
|
94
|
+
if config.respond_to?(setter)
|
95
|
+
config.send(setter, value)
|
96
|
+
end
|
97
|
+
end
|
98
|
+
config
|
99
|
+
end
|
100
|
+
|
101
|
+
def self.configurables
|
102
|
+
@configurables ||= []
|
103
|
+
end
|
104
|
+
|
105
|
+
def self.validations
|
106
|
+
@validations ||= []
|
107
|
+
end
|
108
|
+
|
109
|
+
def self.defaults
|
110
|
+
@defaults ||= {}
|
111
|
+
end
|
112
|
+
|
113
|
+
def self.configurable(setting, default=nil, &validation)
|
114
|
+
attr_accessor(setting)
|
115
|
+
configurables << setting
|
116
|
+
defaults[setting] = default
|
117
|
+
validations << validation if block_given?
|
118
|
+
|
119
|
+
setting
|
120
|
+
end
|
121
|
+
|
122
|
+
configurable :config_file, "/etc/chef/solr.rb" do
|
123
|
+
unless (config_file && File.exist?(config_file) && File.readable?(config_file))
|
124
|
+
log.warn {"* " * 40}
|
125
|
+
log.warn {"Config file #{config_file} does not exist or cannot be read by user (#{Process.euid})"}
|
126
|
+
log.warn {"Default configuration settings will be used"}
|
127
|
+
log.warn {"* " * 40}
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
131
|
+
configurable :index do
|
132
|
+
unless index.nil? # in single-cluster mode, this setting is not required.
|
133
|
+
invalid("You must specify this node's position in the ring as an integer") unless index.kind_of?(Integer)
|
134
|
+
invalid("The index cannot be larger than the cluster size (node-count)") unless (index.to_i <= node_count.to_i)
|
135
|
+
end
|
136
|
+
end
|
137
|
+
|
138
|
+
configurable :node_count, 1 do
|
139
|
+
invalid("You must specify the node_count as an integer") unless node_count.kind_of?(Integer)
|
140
|
+
invalid("The node_count must be 1 or greater") unless node_count >= 1
|
141
|
+
invalid("The node_count cannot be smaller than the index") unless node_count >= index.to_i
|
142
|
+
end
|
143
|
+
|
144
|
+
configurable :ps_tag, ""
|
145
|
+
|
146
|
+
configurable :solr_url, "http://localhost:8983/solr"
|
147
|
+
|
148
|
+
# override the setter for solr_url for backward compatibilty
|
149
|
+
def solr_url=(url)
|
150
|
+
if url && url == "http://localhost:8983"
|
151
|
+
log.warn {"You seem to have a legacy setting for solr_url: did you mean #{url}/solr ?"}
|
152
|
+
url = "#{url}/solr"
|
153
|
+
end
|
154
|
+
@solr_url = url
|
155
|
+
end
|
156
|
+
|
157
|
+
configurable :amqp_host, '0.0.0.0'
|
158
|
+
|
159
|
+
configurable :amqp_port, 5672
|
160
|
+
|
161
|
+
configurable :amqp_user, 'chef'
|
162
|
+
|
163
|
+
configurable :amqp_pass, 'testing'
|
164
|
+
|
165
|
+
configurable :amqp_vhost, '/chef'
|
166
|
+
|
167
|
+
configurable :user, nil
|
168
|
+
|
169
|
+
configurable :group, nil
|
170
|
+
|
171
|
+
configurable :daemonize, false
|
172
|
+
|
173
|
+
alias :daemonize? :daemonize
|
174
|
+
|
175
|
+
configurable :pidfile, DEFAULT_PIDFILE
|
176
|
+
|
177
|
+
def pidfile
|
178
|
+
if @pidfile.equal?(DEFAULT_PIDFILE)
|
179
|
+
Process.euid == 0 ? '/var/run/chef-expander.pid' : '/tmp/chef-expander.pid'
|
180
|
+
else
|
181
|
+
@pidfile
|
182
|
+
end
|
183
|
+
end
|
184
|
+
|
185
|
+
configurable :log_level, :info
|
186
|
+
|
187
|
+
# override the setter for log_level to also actually set the level
|
188
|
+
def log_level=(level)
|
189
|
+
if level #don't accept nil for an answer
|
190
|
+
level = level.to_sym
|
191
|
+
Loggable::LOGGER.level = level
|
192
|
+
@log_level = log_level
|
193
|
+
end
|
194
|
+
level
|
195
|
+
end
|
196
|
+
|
197
|
+
configurable :log_location, STDOUT
|
198
|
+
|
199
|
+
# override the setter for log_location to re-init the logger
|
200
|
+
def log_location=(location)
|
201
|
+
Loggable::LOGGER.init(location) unless location.nil?
|
202
|
+
end
|
203
|
+
|
204
|
+
def initialize
|
205
|
+
reset!
|
206
|
+
end
|
207
|
+
|
208
|
+
def reset!(stdout=nil)
|
209
|
+
self.class.configurables.each do |setting|
|
210
|
+
send("#{setting}=".to_sym, nil)
|
211
|
+
end
|
212
|
+
@stdout = stdout || STDOUT
|
213
|
+
end
|
214
|
+
|
215
|
+
def apply_defaults
|
216
|
+
self.class.defaults.each do |setting, value|
|
217
|
+
self.send("#{setting}=".to_sym, value)
|
218
|
+
end
|
219
|
+
end
|
220
|
+
|
221
|
+
def merge_config(other)
|
222
|
+
self.class.configurables.each do |setting|
|
223
|
+
value = other.send(setting)
|
224
|
+
self.send("#{setting}=".to_sym, value) if value
|
225
|
+
end
|
226
|
+
end
|
227
|
+
|
228
|
+
def fail_if_invalid
|
229
|
+
validate!
|
230
|
+
rescue InvalidConfiguration => e
|
231
|
+
@stdout.puts("Invalid configuration: #{e.message}")
|
232
|
+
exit(1)
|
233
|
+
end
|
234
|
+
|
235
|
+
def invalid(message)
|
236
|
+
raise InvalidConfiguration, message
|
237
|
+
end
|
238
|
+
|
239
|
+
def validate!
|
240
|
+
self.class.validations.each do |validation_proc|
|
241
|
+
instance_eval(&validation_proc)
|
242
|
+
end
|
243
|
+
end
|
244
|
+
|
245
|
+
def vnode_numbers
|
246
|
+
vnodes_per_node = VNODES / node_count
|
247
|
+
lower_bound = (index - 1) * vnodes_per_node
|
248
|
+
upper_bound = lower_bound + vnodes_per_node
|
249
|
+
upper_bound += VNODES % vnodes_per_node if index == node_count
|
250
|
+
(lower_bound...upper_bound).to_a
|
251
|
+
end
|
252
|
+
|
253
|
+
def amqp_config
|
254
|
+
{:host => amqp_host, :port => amqp_port, :user => amqp_user, :pass => amqp_pass, :vhost => amqp_vhost}
|
255
|
+
end
|
256
|
+
|
257
|
+
end
|
258
|
+
|
259
|
+
module CLI
|
260
|
+
@config = Configuration::Base.new
|
261
|
+
|
262
|
+
@option_parser = OptionParser.new do |o|
|
263
|
+
o.banner = "Usage: chef-expander [options]"
|
264
|
+
|
265
|
+
o.on('-c', '--config CONFIG_FILE', 'a configuration file to use') do |conf|
|
266
|
+
@config.config_file = File.expand_path(conf)
|
267
|
+
end
|
268
|
+
|
269
|
+
o.on('-i', '--index INDEX', 'the slot this node will occupy in the ring') do |i|
|
270
|
+
@config.index = i.to_i
|
271
|
+
end
|
272
|
+
|
273
|
+
o.on('-n', '--node-count NUMBER', 'the number of nodes in the ring') do |n|
|
274
|
+
@config.node_count = n.to_i
|
275
|
+
end
|
276
|
+
|
277
|
+
o.on('-l', '--log-level LOG_LEVEL', 'set the log level') do |l|
|
278
|
+
@config.log_level = l
|
279
|
+
end
|
280
|
+
|
281
|
+
o.on('-L', '--logfile LOG_LOCATION', 'Logfile to use') do |l|
|
282
|
+
@config.log_location = l
|
283
|
+
end
|
284
|
+
|
285
|
+
o.on('-d', '--daemonize', 'fork into the background') do
|
286
|
+
@config.daemonize = true
|
287
|
+
end
|
288
|
+
|
289
|
+
o.on('-P', '--pid PIDFILE') do |p|
|
290
|
+
@config.pidfile = p
|
291
|
+
end
|
292
|
+
|
293
|
+
o.on_tail('-h', '--help', 'show this message') do
|
294
|
+
puts "chef-expander #{Expander.version}"
|
295
|
+
puts ''
|
296
|
+
puts o
|
297
|
+
exit 1
|
298
|
+
end
|
299
|
+
|
300
|
+
o.on_tail('-v', '--version', 'show the version and exit') do
|
301
|
+
puts "chef-expander #{Expander.version}"
|
302
|
+
exit 0
|
303
|
+
end
|
304
|
+
|
305
|
+
end
|
306
|
+
|
307
|
+
def self.parse_options(argv)
|
308
|
+
@option_parser.parse!(argv.dup)
|
309
|
+
end
|
310
|
+
|
311
|
+
def self.config
|
312
|
+
@config
|
313
|
+
end
|
314
|
+
|
315
|
+
end
|
316
|
+
|
317
|
+
end
|
318
|
+
|
319
|
+
end
|
320
|
+
end
|
@@ -0,0 +1,206 @@
|
|
1
|
+
#
|
2
|
+
# Author:: Daniel DeLeo (<dan@opscode.com>)
|
3
|
+
# Author:: Seth Falcon (<seth@opscode.com>)
|
4
|
+
# Author:: Chris Walters (<cw@opscode.com>)
|
5
|
+
# Copyright:: Copyright (c) 2010-2011 Opscode, Inc.
|
6
|
+
# License:: Apache License, Version 2.0
|
7
|
+
#
|
8
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
9
|
+
# you may not use this file except in compliance with the License.
|
10
|
+
# You may obtain a copy of the License at
|
11
|
+
#
|
12
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
13
|
+
#
|
14
|
+
# Unless required by applicable law or agreed to in writing, software
|
15
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
16
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
17
|
+
# See the License for the specific language governing permissions and
|
18
|
+
# limitations under the License.
|
19
|
+
#
|
20
|
+
|
21
|
+
require 'bunny'
|
22
|
+
require 'yajl'
|
23
|
+
require 'eventmachine'
|
24
|
+
require 'amqp'
|
25
|
+
require 'mq'
|
26
|
+
require 'highline'
|
27
|
+
|
28
|
+
require 'chef/expander/node'
|
29
|
+
require 'chef/expander/configuration'
|
30
|
+
|
31
|
+
require 'pp'
|
32
|
+
|
33
|
+
module Chef
|
34
|
+
module Expander
|
35
|
+
class Control
|
36
|
+
|
37
|
+
def self.run(argv)
|
38
|
+
remaining_args_after_opts = Expander.init_config(ARGV)
|
39
|
+
new(remaining_args_after_opts).run
|
40
|
+
end
|
41
|
+
|
42
|
+
def self.desc(description)
|
43
|
+
@desc = description
|
44
|
+
end
|
45
|
+
|
46
|
+
def self.option(*args)
|
47
|
+
#TODO
|
48
|
+
end
|
49
|
+
|
50
|
+
def self.arg(*args)
|
51
|
+
#TODO
|
52
|
+
end
|
53
|
+
|
54
|
+
def self.descriptions
|
55
|
+
@descriptions ||= []
|
56
|
+
end
|
57
|
+
|
58
|
+
def self.method_added(method_name)
|
59
|
+
if @desc
|
60
|
+
descriptions << [method_name, method_name.to_s.gsub('_', '-'), @desc]
|
61
|
+
@desc = nil
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
#--
|
66
|
+
# TODO: this is confusing and unneeded. Just whitelist the methods
|
67
|
+
# that map to commands and use +send+
|
68
|
+
def self.compile
|
69
|
+
run_method = "def run; case @argv.first;"
|
70
|
+
descriptions.each do |method_name, command_name, desc|
|
71
|
+
run_method << "when '#{command_name}';#{method_name};"
|
72
|
+
end
|
73
|
+
run_method << "else; help; end; end;"
|
74
|
+
class_eval(run_method, __FILE__, __LINE__)
|
75
|
+
end
|
76
|
+
|
77
|
+
def initialize(argv)
|
78
|
+
@argv = argv.dup
|
79
|
+
end
|
80
|
+
|
81
|
+
desc "Show this message"
|
82
|
+
def help
|
83
|
+
puts "Chef Expander #{Expander.version}"
|
84
|
+
puts "Usage: chef-expanderctl COMMAND"
|
85
|
+
puts
|
86
|
+
puts "Commands:"
|
87
|
+
self.class.descriptions.each do |method_name, command_name, desc|
|
88
|
+
puts " #{command_name}".ljust(15) + desc
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
desc "display the aggregate queue backlog"
|
93
|
+
def queue_depth
|
94
|
+
h = HighLine.new
|
95
|
+
message_counts = []
|
96
|
+
|
97
|
+
amqp_client = Bunny.new(Expander.config.amqp_config)
|
98
|
+
amqp_client.start
|
99
|
+
|
100
|
+
0.upto(VNODES - 1) do |vnode|
|
101
|
+
q = amqp_client.queue("vnode-#{vnode}", :durable => true)
|
102
|
+
message_counts << q.status[:message_count]
|
103
|
+
end
|
104
|
+
total_messages = message_counts.inject(0) { |sum, count| sum + count }
|
105
|
+
max = message_counts.max
|
106
|
+
min = message_counts.min
|
107
|
+
|
108
|
+
avg = total_messages.to_f / message_counts.size.to_f
|
109
|
+
|
110
|
+
puts " total messages: #{total_messages}"
|
111
|
+
puts " average queue depth: #{avg}"
|
112
|
+
puts " max queue depth: #{max}"
|
113
|
+
puts " min queue depth: #{min}"
|
114
|
+
ensure
|
115
|
+
amqp_client.stop if defined?(amqp_client) && amqp_client
|
116
|
+
end
|
117
|
+
|
118
|
+
desc "show the backlog and consumer count for each vnode queue"
|
119
|
+
def queue_status
|
120
|
+
h = HighLine.new
|
121
|
+
queue_status = [h.color("VNode", :bold), h.color("Messages", :bold), h.color("Consumers", :bold)]
|
122
|
+
|
123
|
+
total_messages = 0
|
124
|
+
|
125
|
+
amqp_client = Bunny.new(Expander.config.amqp_config)
|
126
|
+
amqp_client.start
|
127
|
+
|
128
|
+
0.upto(VNODES - 1) do |vnode|
|
129
|
+
q = amqp_client.queue("vnode-#{vnode}", :durable => true)
|
130
|
+
status = q.status
|
131
|
+
# returns {:message_count => method.message_count, :consumer_count => method.consumer_count}
|
132
|
+
queue_status << vnode.to_s << status[:message_count].to_s << status[:consumer_count].to_s
|
133
|
+
total_messages += status[:message_count]
|
134
|
+
end
|
135
|
+
puts " total messages: #{total_messages}"
|
136
|
+
puts
|
137
|
+
puts h.list(queue_status, :columns_across, 3)
|
138
|
+
ensure
|
139
|
+
amqp_client.stop if defined?(amqp_client) && amqp_client
|
140
|
+
end
|
141
|
+
|
142
|
+
desc "show the status of the nodes in the cluster"
|
143
|
+
def node_status
|
144
|
+
status_mutex = Mutex.new
|
145
|
+
h = ::HighLine.new
|
146
|
+
node_status = [h.color("Host", :bold), h.color("PID", :bold), h.color("GUID", :bold), h.color("Vnodes", :bold)]
|
147
|
+
|
148
|
+
print("Collecting status info from the cluster...")
|
149
|
+
|
150
|
+
AMQP.start(Expander.config.amqp_config) do
|
151
|
+
node = Expander::Node.local_node
|
152
|
+
node.exclusive_control_queue.subscribe do |header, message|
|
153
|
+
status = Yajl::Parser.parse(message)
|
154
|
+
status_mutex.synchronize do
|
155
|
+
node_status << status["hostname_f"]
|
156
|
+
node_status << status["pid"].to_s
|
157
|
+
node_status << status["guid"]
|
158
|
+
# BIG ASSUMPTION HERE that nodes only have contiguous vnode ranges
|
159
|
+
# will not be true once vnode recovery is implemented
|
160
|
+
node_status << "#{status["vnodes"].min}-#{status["vnodes"].max}"
|
161
|
+
end
|
162
|
+
end
|
163
|
+
node.broadcast_message(Yajl::Encoder.encode(:action => :status, :rsvp => node.exclusive_control_queue_name))
|
164
|
+
EM.add_timer(2) { AMQP.stop;EM.stop }
|
165
|
+
end
|
166
|
+
|
167
|
+
puts "done"
|
168
|
+
puts
|
169
|
+
puts h.list(node_status, :columns_across, 4)
|
170
|
+
puts
|
171
|
+
end
|
172
|
+
|
173
|
+
desc "sets the log level of all nodes in the cluster"
|
174
|
+
def log_level
|
175
|
+
@argv.shift
|
176
|
+
level = @argv.first
|
177
|
+
acceptable_levels = %w{debug info warn error fatal}
|
178
|
+
unless acceptable_levels.include?(level)
|
179
|
+
puts "Log level must be one of #{acceptable_levels.join(', ')}"
|
180
|
+
exit 1
|
181
|
+
end
|
182
|
+
|
183
|
+
h = HighLine.new
|
184
|
+
response_mutex = Mutex.new
|
185
|
+
|
186
|
+
responses = [h.color("Host", :bold), h.color("PID", :bold), h.color("GUID", :bold), h.color("Log Level", :bold)]
|
187
|
+
AMQP.start(Expander.config.amqp_config) do
|
188
|
+
node = Expander::Node.local_node
|
189
|
+
node.exclusive_control_queue.subscribe do |header, message|
|
190
|
+
reply = Yajl::Parser.parse(message)
|
191
|
+
n = reply['node']
|
192
|
+
response_mutex.synchronize do
|
193
|
+
responses << n["hostname_f"] << n["pid"].to_s << n["guid"] << reply["level"]
|
194
|
+
end
|
195
|
+
end
|
196
|
+
node.broadcast_message(Yajl::Encoder.encode({:action => :set_log_level, :level => level, :rsvp => node.exclusive_control_queue_name}))
|
197
|
+
EM.add_timer(2) { AMQP.stop; EM.stop }
|
198
|
+
end
|
199
|
+
puts h.list(responses, :columns_across, 4)
|
200
|
+
end
|
201
|
+
|
202
|
+
|
203
|
+
compile
|
204
|
+
end
|
205
|
+
end
|
206
|
+
end
|