samhendley-god 0.7.13
Sign up to get free protection for your applications and to get access to all the features.
- data/History.txt +293 -0
- data/Manifest.txt +114 -0
- data/README.txt +60 -0
- data/Rakefile +35 -0
- data/bin/god +128 -0
- data/examples/events.god +84 -0
- data/examples/gravatar.god +54 -0
- data/examples/single.god +66 -0
- data/ext/god/extconf.rb +55 -0
- data/ext/god/kqueue_handler.c +123 -0
- data/ext/god/netlink_handler.c +167 -0
- data/init/god +42 -0
- data/lib/god.rb +667 -0
- data/lib/god/behavior.rb +52 -0
- data/lib/god/behaviors/clean_pid_file.rb +21 -0
- data/lib/god/behaviors/clean_unix_socket.rb +21 -0
- data/lib/god/behaviors/notify_when_flapping.rb +51 -0
- data/lib/god/cli/command.rb +229 -0
- data/lib/god/cli/run.rb +176 -0
- data/lib/god/cli/version.rb +23 -0
- data/lib/god/condition.rb +96 -0
- data/lib/god/conditions/always.rb +23 -0
- data/lib/god/conditions/complex.rb +86 -0
- data/lib/god/conditions/cpu_usage.rb +80 -0
- data/lib/god/conditions/degrading_lambda.rb +52 -0
- data/lib/god/conditions/disk_usage.rb +27 -0
- data/lib/god/conditions/file_mtime.rb +28 -0
- data/lib/god/conditions/flapping.rb +128 -0
- data/lib/god/conditions/http_response_code.rb +168 -0
- data/lib/god/conditions/lambda.rb +25 -0
- data/lib/god/conditions/memory_usage.rb +82 -0
- data/lib/god/conditions/process_exits.rb +72 -0
- data/lib/god/conditions/process_running.rb +74 -0
- data/lib/god/conditions/tries.rb +44 -0
- data/lib/god/configurable.rb +57 -0
- data/lib/god/contact.rb +106 -0
- data/lib/god/contacts/campfire.rb +82 -0
- data/lib/god/contacts/email.rb +95 -0
- data/lib/god/contacts/jabber.rb +65 -0
- data/lib/god/contacts/twitter.rb +39 -0
- data/lib/god/contacts/webhook.rb +47 -0
- data/lib/god/dependency_graph.rb +41 -0
- data/lib/god/diagnostics.rb +37 -0
- data/lib/god/driver.rb +206 -0
- data/lib/god/errors.rb +24 -0
- data/lib/god/event_handler.rb +111 -0
- data/lib/god/event_handlers/dummy_handler.rb +13 -0
- data/lib/god/event_handlers/kqueue_handler.rb +17 -0
- data/lib/god/event_handlers/netlink_handler.rb +13 -0
- data/lib/god/logger.rb +120 -0
- data/lib/god/metric.rb +59 -0
- data/lib/god/process.rb +342 -0
- data/lib/god/registry.rb +32 -0
- data/lib/god/simple_logger.rb +53 -0
- data/lib/god/socket.rb +96 -0
- data/lib/god/sugar.rb +47 -0
- data/lib/god/system/portable_poller.rb +42 -0
- data/lib/god/system/process.rb +42 -0
- data/lib/god/system/slash_proc_poller.rb +92 -0
- data/lib/god/task.rb +491 -0
- data/lib/god/timeline.rb +25 -0
- data/lib/god/trigger.rb +43 -0
- data/lib/god/watch.rb +184 -0
- data/test/configs/child_events/child_events.god +44 -0
- data/test/configs/child_events/simple_server.rb +3 -0
- data/test/configs/child_polls/child_polls.god +37 -0
- data/test/configs/child_polls/simple_server.rb +12 -0
- data/test/configs/complex/complex.god +59 -0
- data/test/configs/complex/simple_server.rb +3 -0
- data/test/configs/contact/contact.god +84 -0
- data/test/configs/contact/simple_server.rb +3 -0
- data/test/configs/daemon_events/daemon_events.god +37 -0
- data/test/configs/daemon_events/simple_server.rb +8 -0
- data/test/configs/daemon_events/simple_server_stop.rb +11 -0
- data/test/configs/daemon_polls/daemon_polls.god +17 -0
- data/test/configs/daemon_polls/simple_server.rb +6 -0
- data/test/configs/degrading_lambda/degrading_lambda.god +31 -0
- data/test/configs/degrading_lambda/tcp_server.rb +15 -0
- data/test/configs/matias/matias.god +50 -0
- data/test/configs/real.rb +59 -0
- data/test/configs/running_load/running_load.god +16 -0
- data/test/configs/stress/simple_server.rb +3 -0
- data/test/configs/stress/stress.god +15 -0
- data/test/configs/task/logs/.placeholder +0 -0
- data/test/configs/task/task.god +26 -0
- data/test/configs/test.rb +61 -0
- data/test/helper.rb +151 -0
- data/test/suite.rb +6 -0
- data/test/test_behavior.rb +21 -0
- data/test/test_campfire.rb +41 -0
- data/test/test_condition.rb +50 -0
- data/test/test_conditions_disk_usage.rb +56 -0
- data/test/test_conditions_http_response_code.rb +109 -0
- data/test/test_conditions_process_running.rb +44 -0
- data/test/test_conditions_tries.rb +67 -0
- data/test/test_contact.rb +109 -0
- data/test/test_dependency_graph.rb +62 -0
- data/test/test_driver.rb +11 -0
- data/test/test_email.rb +45 -0
- data/test/test_event_handler.rb +80 -0
- data/test/test_god.rb +598 -0
- data/test/test_handlers_kqueue_handler.rb +16 -0
- data/test/test_logger.rb +63 -0
- data/test/test_metric.rb +72 -0
- data/test/test_process.rb +246 -0
- data/test/test_registry.rb +15 -0
- data/test/test_socket.rb +42 -0
- data/test/test_sugar.rb +42 -0
- data/test/test_system_portable_poller.rb +17 -0
- data/test/test_system_process.rb +30 -0
- data/test/test_task.rb +262 -0
- data/test/test_timeline.rb +37 -0
- data/test/test_trigger.rb +59 -0
- data/test/test_watch.rb +279 -0
- metadata +193 -0
@@ -0,0 +1,57 @@
|
|
1
|
+
module God
|
2
|
+
|
3
|
+
module Configurable
|
4
|
+
# Override this method in your Configurable (optional)
|
5
|
+
#
|
6
|
+
# Called once after the Configurable has been sent to the block and attributes have been
|
7
|
+
# set. Do any post-processing on attributes here
|
8
|
+
def prepare
|
9
|
+
|
10
|
+
end
|
11
|
+
|
12
|
+
def reset
|
13
|
+
|
14
|
+
end
|
15
|
+
|
16
|
+
# Override this method in your Configurable (optional)
|
17
|
+
#
|
18
|
+
# Called once during evaluation of the config file. Return true if valid, false otherwise
|
19
|
+
#
|
20
|
+
# A convenience method 'complain' is available that will print out a message and return false,
|
21
|
+
# making it easy to report multiple validation errors:
|
22
|
+
#
|
23
|
+
# def valid?
|
24
|
+
# valid = true
|
25
|
+
# valid &= complain("You must specify the 'pid_file' attribute for :memory_usage") if self.pid_file.nil?
|
26
|
+
# valid &= complain("You must specify the 'above' attribute for :memory_usage") if self.above.nil?
|
27
|
+
# valid
|
28
|
+
# end
|
29
|
+
def valid?
|
30
|
+
true
|
31
|
+
end
|
32
|
+
|
33
|
+
def base_name
|
34
|
+
x = 1 # fix for MRI's local scope optimization bug DO NOT REMOVE!
|
35
|
+
self.class.name.split('::').last
|
36
|
+
end
|
37
|
+
|
38
|
+
def friendly_name
|
39
|
+
base_name
|
40
|
+
end
|
41
|
+
|
42
|
+
def self.complain(text, c = nil)
|
43
|
+
watch = c.watch rescue nil
|
44
|
+
msg = ""
|
45
|
+
msg += "#{watch.name}: " if watch
|
46
|
+
msg += text
|
47
|
+
msg += " for #{c.friendly_name}" if c
|
48
|
+
applog(watch, :error, msg)
|
49
|
+
false
|
50
|
+
end
|
51
|
+
|
52
|
+
def complain(text, c = nil)
|
53
|
+
Configurable.complain(text, c)
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
end
|
data/lib/god/contact.rb
ADDED
@@ -0,0 +1,106 @@
|
|
1
|
+
module God
|
2
|
+
|
3
|
+
class Contact
|
4
|
+
include Configurable
|
5
|
+
|
6
|
+
attr_accessor :name, :group, :info
|
7
|
+
|
8
|
+
def self.generate(kind)
|
9
|
+
sym = kind.to_s.capitalize.gsub(/_(.)/){$1.upcase}.intern
|
10
|
+
c = God::Contacts.const_get(sym).new
|
11
|
+
|
12
|
+
unless c.kind_of?(Contact)
|
13
|
+
abort "Contact '#{c.class.name}' must subclass God::Contact"
|
14
|
+
end
|
15
|
+
|
16
|
+
c
|
17
|
+
rescue NameError
|
18
|
+
raise NoSuchContactError.new("No Contact found with the class name God::Contacts::#{sym}")
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.valid?(contact)
|
22
|
+
valid = true
|
23
|
+
valid &= Configurable.complain("Attribute 'name' must be specified", contact) if contact.name.nil?
|
24
|
+
valid
|
25
|
+
end
|
26
|
+
|
27
|
+
# Normalize the given notify specification into canonical form.
|
28
|
+
# +spec+ is the notify spec as a String, Array of Strings, or Hash
|
29
|
+
#
|
30
|
+
# Canonical form looks like:
|
31
|
+
# {:contacts => ['fred', 'john'], :priority => '1', :category => 'awesome'}
|
32
|
+
# Where :contacts will be present and point to an Array of Strings. Both
|
33
|
+
# :priority and :category may not be present but if they are, they will each
|
34
|
+
# contain a single String.
|
35
|
+
#
|
36
|
+
# Returns normalized notify spec
|
37
|
+
# Raises ArgumentError on invalid spec (message contains details)
|
38
|
+
def self.normalize(spec)
|
39
|
+
case spec
|
40
|
+
when String
|
41
|
+
{:contacts => Array(spec)}
|
42
|
+
when Array
|
43
|
+
unless spec.select { |x| !x.instance_of?(String) }.empty?
|
44
|
+
raise ArgumentError.new("contains non-String elements")
|
45
|
+
end
|
46
|
+
{:contacts => spec}
|
47
|
+
when Hash
|
48
|
+
copy = spec.dup
|
49
|
+
|
50
|
+
# check :contacts
|
51
|
+
if contacts = copy.delete(:contacts)
|
52
|
+
case contacts
|
53
|
+
when String
|
54
|
+
# valid
|
55
|
+
when Array
|
56
|
+
unless contacts.select { |x| !x.instance_of?(String) }.empty?
|
57
|
+
raise ArgumentError.new("has a :contacts key containing non-String elements")
|
58
|
+
end
|
59
|
+
# valid
|
60
|
+
else
|
61
|
+
raise ArgumentError.new("must have a :contacts key pointing to a String or Array of Strings")
|
62
|
+
end
|
63
|
+
else
|
64
|
+
raise ArgumentError.new("must have a :contacts key")
|
65
|
+
end
|
66
|
+
|
67
|
+
# remove priority and category
|
68
|
+
copy.delete(:priority)
|
69
|
+
copy.delete(:category)
|
70
|
+
|
71
|
+
# check for invalid keys
|
72
|
+
unless copy.empty?
|
73
|
+
raise ArgumentError.new("contains extra elements: #{copy.inspect}")
|
74
|
+
end
|
75
|
+
|
76
|
+
# normalize
|
77
|
+
spec[:contacts] &&= Array(spec[:contacts])
|
78
|
+
spec[:priority] &&= spec[:priority].to_s
|
79
|
+
spec[:category] &&= spec[:category].to_s
|
80
|
+
|
81
|
+
spec
|
82
|
+
else
|
83
|
+
raise ArgumentError.new("must be a String (contact name), Array (of contact names), or Hash (contact specification)")
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
# Abstract
|
88
|
+
# Send the message to the external source
|
89
|
+
# +message+ is the message body returned from the condition
|
90
|
+
# +time+ is the Time at which the notification was made
|
91
|
+
# +priority+ is the arbitrary priority String
|
92
|
+
# +category+ is the arbitrary category String
|
93
|
+
# +host+ is the hostname of the server
|
94
|
+
def notify(message, time, priority, category, host)
|
95
|
+
raise AbstractMethodNotOverriddenError.new("Contact#notify must be overridden in subclasses")
|
96
|
+
end
|
97
|
+
|
98
|
+
# Construct the friendly name of this Contact, looks like:
|
99
|
+
#
|
100
|
+
# Contact FooBar
|
101
|
+
def friendly_name
|
102
|
+
super + " Contact '#{self.name}'"
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
end
|
@@ -0,0 +1,82 @@
|
|
1
|
+
# notify campfire using tinder http://tinder.rubyforge.org
|
2
|
+
#
|
3
|
+
# Example: set up a new campfire notifier
|
4
|
+
#
|
5
|
+
# Credentials
|
6
|
+
#
|
7
|
+
# God::Contacts::Campfire.server_settings = {
|
8
|
+
# :subdomain => "yoursubdomain",
|
9
|
+
# :user_name => "youruser",
|
10
|
+
# :room => "yourroom",
|
11
|
+
# :password => "yourpassword"
|
12
|
+
# }
|
13
|
+
#
|
14
|
+
# Register a new notifier
|
15
|
+
#
|
16
|
+
# God.contact(:campfire) do |c|
|
17
|
+
# c.name = 'campfire'
|
18
|
+
# end
|
19
|
+
#
|
20
|
+
# Define a transition for the process running event
|
21
|
+
#
|
22
|
+
# w.transition(:up, :start) do |on|
|
23
|
+
# on.condition(:process_running) do |c|
|
24
|
+
# c.running = true
|
25
|
+
# c.notify = 'campfire'
|
26
|
+
# end
|
27
|
+
# end
|
28
|
+
|
29
|
+
require 'tinder'
|
30
|
+
|
31
|
+
module God
|
32
|
+
module Contacts
|
33
|
+
|
34
|
+
class Campfire < Contact
|
35
|
+
class << self
|
36
|
+
attr_accessor :server_settings, :format
|
37
|
+
end
|
38
|
+
|
39
|
+
self.server_settings = {:subdomain => '',
|
40
|
+
:user_name => '',
|
41
|
+
:password => '',
|
42
|
+
:room => ''}
|
43
|
+
|
44
|
+
self.format = lambda do |message, host|
|
45
|
+
<<-EOF
|
46
|
+
#{host} - #{message}
|
47
|
+
EOF
|
48
|
+
end
|
49
|
+
|
50
|
+
def initialize
|
51
|
+
@room = nil
|
52
|
+
end
|
53
|
+
|
54
|
+
def notify(message, time, priority, category, host)
|
55
|
+
begin
|
56
|
+
body = Campfire.format.call(message,host)
|
57
|
+
|
58
|
+
room.speak body
|
59
|
+
|
60
|
+
self.info = "notified campfire: #{Campfire.server_settings[:subdomain]}"
|
61
|
+
rescue => e
|
62
|
+
applog(nil, :info, "failed to notify campfire: #{e.message}")
|
63
|
+
applog(nil, :debug, e.backtrace.join("\n"))
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
private
|
68
|
+
|
69
|
+
def room
|
70
|
+
unless @room
|
71
|
+
applog(nil,:debug, "initializing campfire connection using credentials: #{Campfire.server_settings.inspect}")
|
72
|
+
|
73
|
+
campfire = Tinder::Campfire.new Campfire.server_settings[:subdomain]
|
74
|
+
campfire.login Campfire.server_settings[:user_name], Campfire.server_settings[:password]
|
75
|
+
@room = campfire.find_room_by_name(Campfire.server_settings[:room])
|
76
|
+
end
|
77
|
+
@room
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
end
|
82
|
+
end
|
@@ -0,0 +1,95 @@
|
|
1
|
+
require 'time'
|
2
|
+
require 'net/smtp'
|
3
|
+
|
4
|
+
module God
|
5
|
+
module Contacts
|
6
|
+
|
7
|
+
class Email < Contact
|
8
|
+
class << self
|
9
|
+
attr_accessor :message_settings, :delivery_method, :server_settings, :sendmail_settings, :format
|
10
|
+
end
|
11
|
+
|
12
|
+
self.message_settings = {:from => 'god@example.com'}
|
13
|
+
|
14
|
+
self.delivery_method = :smtp # or :sendmail
|
15
|
+
|
16
|
+
self.server_settings = {:address => 'localhost',
|
17
|
+
:port => 25}
|
18
|
+
# :domain
|
19
|
+
# :user_name
|
20
|
+
# :password
|
21
|
+
# :authentication
|
22
|
+
|
23
|
+
self.sendmail_settings = {:location => '/usr/sbin/sendmail',
|
24
|
+
:arguments => '-i -t'
|
25
|
+
}
|
26
|
+
|
27
|
+
self.format = lambda do |name, email, message, time, priority, category, host|
|
28
|
+
<<-EOF
|
29
|
+
From: god <#{self.message_settings[:from]}>
|
30
|
+
To: #{name} <#{email}>
|
31
|
+
Subject: [god] #{message}
|
32
|
+
Date: #{Time.now.httpdate}
|
33
|
+
Message-Id: <unique.message.id.string@example.com>
|
34
|
+
|
35
|
+
Message: #{message}
|
36
|
+
Host: #{host}
|
37
|
+
Priority: #{priority}
|
38
|
+
Category: #{category}
|
39
|
+
EOF
|
40
|
+
end
|
41
|
+
|
42
|
+
attr_accessor :email
|
43
|
+
|
44
|
+
def valid?
|
45
|
+
valid = true
|
46
|
+
valid &= complain("Attribute 'email' must be specified", self) if self.email.nil?
|
47
|
+
valid
|
48
|
+
end
|
49
|
+
|
50
|
+
def notify(message, time, priority, category, host)
|
51
|
+
begin
|
52
|
+
body = Email.format.call(self.name, self.email, message, time, priority, category, host)
|
53
|
+
|
54
|
+
case Email.delivery_method
|
55
|
+
when :smtp
|
56
|
+
notify_smtp(body)
|
57
|
+
when :sendmail
|
58
|
+
notify_sendmail(body)
|
59
|
+
else
|
60
|
+
raise "unknown delivery method: #{Email.delivery_method}"
|
61
|
+
end
|
62
|
+
|
63
|
+
self.info = "sent email to #{self.email}"
|
64
|
+
rescue => e
|
65
|
+
applog(nil, :info, "failed to send email to #{self.email}: #{e.message}")
|
66
|
+
applog(nil, :debug, e.backtrace.join("\n"))
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
# private
|
71
|
+
|
72
|
+
def notify_smtp(mail)
|
73
|
+
args = [Email.server_settings[:address], Email.server_settings[:port]]
|
74
|
+
if Email.server_settings[:authentication]
|
75
|
+
args << Email.server_settings[:domain]
|
76
|
+
args << Email.server_settings[:user_name]
|
77
|
+
args << Email.server_settings[:password]
|
78
|
+
args << Email.server_settings[:authentication]
|
79
|
+
end
|
80
|
+
|
81
|
+
Net::SMTP.start(*args) do |smtp|
|
82
|
+
smtp.send_message mail, Email.message_settings[:from], self.email
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
def notify_sendmail(mail)
|
87
|
+
IO.popen("#{Email.sendmail_settings[:location]} #{Email.sendmail_settings[:arguments]}","w+") do |sm|
|
88
|
+
sm.print(mail.gsub(/\r/, ''))
|
89
|
+
sm.flush
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
end
|
95
|
+
end
|
@@ -0,0 +1,65 @@
|
|
1
|
+
# To add Jabber notifications you must have xmpp4r gem installed.
|
2
|
+
# Configure your watches like this:
|
3
|
+
#
|
4
|
+
# God::Contacts::Jabber.settings = { :jabber_id => 'sender@example.com',
|
5
|
+
# :password => 'secret' }
|
6
|
+
# God.contact(:jabber) do |c|
|
7
|
+
# c.name = 'Tester'
|
8
|
+
# c.jabber_id = 'receiver@example.com'
|
9
|
+
# c.group = 'developers'
|
10
|
+
# end
|
11
|
+
|
12
|
+
module XMPP4R
|
13
|
+
require 'rubygems'
|
14
|
+
require 'xmpp4r'
|
15
|
+
include Jabber
|
16
|
+
end
|
17
|
+
|
18
|
+
module God
|
19
|
+
module Contacts
|
20
|
+
class Jabber < Contact
|
21
|
+
class << self
|
22
|
+
attr_accessor :settings, :format
|
23
|
+
end
|
24
|
+
|
25
|
+
self.format = lambda do |message, priority, category, host|
|
26
|
+
text = "Message: #{message}\n"
|
27
|
+
text += "Host: #{host}\n" if host
|
28
|
+
text += "Priority: #{priority}\n" if priority
|
29
|
+
text += "Category: #{category}\n" if category
|
30
|
+
return text
|
31
|
+
end
|
32
|
+
|
33
|
+
attr_accessor :jabber_id
|
34
|
+
|
35
|
+
def valid?
|
36
|
+
valid = true
|
37
|
+
end
|
38
|
+
|
39
|
+
def notify(message, time, priority, category, host)
|
40
|
+
begin
|
41
|
+
jabber_id = XMPP4R::JID::new "#{Jabber.settings[:jabber_id]}/God"
|
42
|
+
jabber_client = XMPP4R::Client::new jabber_id
|
43
|
+
jabber_client.connect
|
44
|
+
jabber_client.auth Jabber.settings[:password]
|
45
|
+
|
46
|
+
body = Jabber.format.call message, priority, category, host
|
47
|
+
|
48
|
+
message = XMPP4R::Message::new self.jabber_id, body
|
49
|
+
message.set_type :normal
|
50
|
+
message.set_id '1'
|
51
|
+
message.set_subject 'God'
|
52
|
+
jabber_client.send message
|
53
|
+
|
54
|
+
self.info = "sent jabber message to #{self.jabber_id}"
|
55
|
+
rescue => e
|
56
|
+
puts e.message
|
57
|
+
puts e.backtrace.join("\n")
|
58
|
+
|
59
|
+
self.info = "failed to send jabber message to #{self.jabber_id}: #{e.message}"
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
end
|
65
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
# For Twitter updates you need the 'twitter' gem
|
2
|
+
# (gem install twitter)
|
3
|
+
#
|
4
|
+
# Configure your watches like this:
|
5
|
+
#
|
6
|
+
# God::Contacts::Twitter.settings = { :username => 'sender@example.com',
|
7
|
+
# :password => 'secret' }
|
8
|
+
# God.contact(:twitter) do |c|
|
9
|
+
# c.name = 'Tester'
|
10
|
+
# c.group = 'developers'
|
11
|
+
# end
|
12
|
+
|
13
|
+
require 'rubygems'
|
14
|
+
require 'twitter'
|
15
|
+
|
16
|
+
module God
|
17
|
+
module Contacts
|
18
|
+
class Twitter < Contact
|
19
|
+
class << self
|
20
|
+
attr_accessor :settings
|
21
|
+
end
|
22
|
+
|
23
|
+
def valid?
|
24
|
+
valid = true
|
25
|
+
end
|
26
|
+
|
27
|
+
def notify(message, time, priority, category, host)
|
28
|
+
begin
|
29
|
+
::Twitter::Base.new(Twitter.settings[:username],
|
30
|
+
Twitter.settings[:password]).update(message)
|
31
|
+
|
32
|
+
self.info = "sent twitter update as #{Twitter.settings[:username]}"
|
33
|
+
rescue => e
|
34
|
+
self.info = "failed to send twitter update from #{self.twitter_id}: #{e.message}"
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|