poise-monit 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +11 -0
- data/.kitchen.yml +8 -0
- data/.travis.yml +20 -0
- data/.yardopts +7 -0
- data/Berksfile +29 -0
- data/CHANGELOG.md +5 -0
- data/Gemfile +35 -0
- data/LICENSE +201 -0
- data/README.md +242 -0
- data/Rakefile +17 -0
- data/chef/attributes/default.rb +37 -0
- data/chef/recipes/default.rb +22 -0
- data/chef/templates/monit.conf.erb +27 -0
- data/chef/templates/monit_service.conf.erb +6 -0
- data/lib/poise_monit.rb +24 -0
- data/lib/poise_monit/cheftie.rb +19 -0
- data/lib/poise_monit/error.rb +21 -0
- data/lib/poise_monit/monit_providers.rb +36 -0
- data/lib/poise_monit/monit_providers/base.rb +173 -0
- data/lib/poise_monit/monit_providers/binaries.rb +108 -0
- data/lib/poise_monit/monit_providers/dummy.rb +57 -0
- data/lib/poise_monit/monit_providers/system.rb +86 -0
- data/lib/poise_monit/resources.rb +28 -0
- data/lib/poise_monit/resources/monit.rb +238 -0
- data/lib/poise_monit/resources/monit_config.rb +111 -0
- data/lib/poise_monit/resources/monit_service.rb +194 -0
- data/lib/poise_monit/resources/monit_test.rb +127 -0
- data/lib/poise_monit/service_providers.rb +26 -0
- data/lib/poise_monit/service_providers/monit.rb +124 -0
- data/lib/poise_monit/version.rb +20 -0
- data/poise-monit.gemspec +46 -0
- data/test/cookbooks/poise-monit_test/attributes/default.rb +17 -0
- data/test/cookbooks/poise-monit_test/metadata.rb +19 -0
- data/test/cookbooks/poise-monit_test/recipes/default.rb +31 -0
- data/test/docker/docker.ca +29 -0
- data/test/docker/docker.pem +83 -0
- data/test/gemfiles/chef-12.gemfile +19 -0
- data/test/gemfiles/master.gemfile +24 -0
- data/test/integration/default/serverspec/Gemfile +19 -0
- data/test/integration/default/serverspec/default_spec.rb +67 -0
- data/test/spec/monit_providers/binaries_spec.rb +95 -0
- data/test/spec/monit_providers/dummy_spec.rb +52 -0
- data/test/spec/monit_providers/system_spec.rb +74 -0
- data/test/spec/recipe_spec.rb +40 -0
- data/test/spec/resources/monit_config_spec.rb +43 -0
- data/test/spec/resources/monit_service_spec.rb +297 -0
- data/test/spec/resources/monit_spec.rb +101 -0
- data/test/spec/service_providers/monit_spec.rb +58 -0
- data/test/spec/spec_helper.rb +19 -0
- metadata +182 -0
@@ -0,0 +1,194 @@
|
|
1
|
+
#
|
2
|
+
# Copyright 2015, Noah Kantrowitz
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'chef/resource/service'
|
18
|
+
require 'chef/provider/service'
|
19
|
+
require 'poise'
|
20
|
+
|
21
|
+
|
22
|
+
module PoiseMonit
|
23
|
+
module Resources
|
24
|
+
# (see MonitService::Resource)
|
25
|
+
# @since 1.0.0
|
26
|
+
module MonitService
|
27
|
+
# Values from `monit status` that mean the service is disabled.
|
28
|
+
DISABLED_STATUSES = /^Not monitored$/
|
29
|
+
# Values from `monit status` that mean the service is running.
|
30
|
+
RUNNING_STATUSES = /^(Accessible|Running|Online with all services|Status ok|UP)$/
|
31
|
+
# Value from monit action subcommands that mean the service doesn't exist.
|
32
|
+
NO_SERVICE_ERROR = /There is no service/
|
33
|
+
|
34
|
+
# Default time to wait for a monit command to succeed.
|
35
|
+
DEFAULT_TIMEOUT = 20
|
36
|
+
# Default time to sleep between tries.
|
37
|
+
DEFAULT_WAIT = 1
|
38
|
+
|
39
|
+
# A `monit_service` resource to control Monit-based services.
|
40
|
+
#
|
41
|
+
# @provides monit_service
|
42
|
+
# @action enable
|
43
|
+
# @action disable
|
44
|
+
# @action start
|
45
|
+
# @action stop
|
46
|
+
# @action restart
|
47
|
+
# @example
|
48
|
+
# monit_service 'httpd'
|
49
|
+
class Resource < Chef::Resource::Service
|
50
|
+
include Poise(parent: :monit)
|
51
|
+
provides(:monit_service)
|
52
|
+
actions(:enable, :disable, :start, :stop, :restart)
|
53
|
+
|
54
|
+
attribute(:monit_config_path, kind_of: [String, NilClass, FalseClass])
|
55
|
+
|
56
|
+
# Unsupported properties.
|
57
|
+
%w{pattern reload_command priority timeout parameters run_levels}.each do |name|
|
58
|
+
define_method(name) do |*args|
|
59
|
+
raise NoMethodError.new("Property #{name} is not supported on monit_service")
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
# Lie about supports.
|
64
|
+
# @api private
|
65
|
+
def supports(arg={})
|
66
|
+
{restart: true}
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
# The provider for `monit_service`.
|
71
|
+
#
|
72
|
+
# @see Resource
|
73
|
+
# @provides monit_service
|
74
|
+
class Provider < Chef::Provider::Service
|
75
|
+
include Poise
|
76
|
+
provides(:monit_service)
|
77
|
+
|
78
|
+
def load_current_resource
|
79
|
+
super
|
80
|
+
@current_resource = MonitService::Resource.new(new_resource.name).tap do |r|
|
81
|
+
r.service_name(new_resource.service_name)
|
82
|
+
if new_resource.monit_config_path && !::File.exist?(new_resource.monit_config_path)
|
83
|
+
Chef::Log.debug("[#{new_resource}] Config file #{new_resource.monit_config_path} does not exist, not checking status")
|
84
|
+
r.enabled(false)
|
85
|
+
r.running(false)
|
86
|
+
else
|
87
|
+
Chef::Log.debug("[#{new_resource}] Checking status for #{new_resource.service_name}")
|
88
|
+
status = find_monit_status
|
89
|
+
Chef::Log.debug("[#{new_resource}] Status is #{status.inspect}")
|
90
|
+
case status
|
91
|
+
when nil, false
|
92
|
+
# Unable to find a status at all.
|
93
|
+
r.enabled(false)
|
94
|
+
r.running(false)
|
95
|
+
when /^Does not exist/
|
96
|
+
# It is monitored but we don't know the status yet, assume the
|
97
|
+
# worst (run start and stop always).
|
98
|
+
r.enabled(true)
|
99
|
+
r.running(self.action != :start)
|
100
|
+
when DISABLED_STATUSES
|
101
|
+
r.enabled(false)
|
102
|
+
# It could be running, but we don't know.
|
103
|
+
r.running(false)
|
104
|
+
when RUNNING_STATUSES
|
105
|
+
r.enabled(true)
|
106
|
+
r.running(true)
|
107
|
+
else
|
108
|
+
r.enabled(true)
|
109
|
+
r.running(false)
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
private
|
116
|
+
|
117
|
+
def enable_service
|
118
|
+
monit_shell_out!('monitor')
|
119
|
+
end
|
120
|
+
|
121
|
+
def disable_service
|
122
|
+
if new_resource.monit_config_path && !::File.exist?(new_resource.monit_config_path)
|
123
|
+
Chef::Log.debug("[#{new_resource}] Config file #{new_resource.monit_config_path} does not exist, not trying to unmonitor")
|
124
|
+
return
|
125
|
+
end
|
126
|
+
monit_shell_out!('unmonitor') do |cmd|
|
127
|
+
# Command fails if it has an error and does not include the service
|
128
|
+
# error message.
|
129
|
+
cmd.error? && cmd.stdout !~ NO_SERVICE_ERROR && cmd.stderr !~ NO_SERVICE_ERROR
|
130
|
+
end
|
131
|
+
end
|
132
|
+
|
133
|
+
def start_service
|
134
|
+
monit_shell_out!('start')
|
135
|
+
monit_shell_out!('start')
|
136
|
+
end
|
137
|
+
|
138
|
+
def stop_service
|
139
|
+
if new_resource.monit_config_path && !::File.exist?(new_resource.monit_config_path)
|
140
|
+
Chef::Log.debug("[#{new_resource}] Config file #{new_resource.monit_config_path} does not exist, not trying to stop")
|
141
|
+
return
|
142
|
+
end
|
143
|
+
monit_shell_out!('stop') do |cmd|
|
144
|
+
# Command fails if it has an error and does not include the service
|
145
|
+
# error message. Then check that it is really stopped.
|
146
|
+
cmd.error? && cmd.stdout !~ NO_SERVICE_ERROR && cmd.stderr !~ NO_SERVICE_ERROR
|
147
|
+
end
|
148
|
+
end
|
149
|
+
|
150
|
+
def restart_service
|
151
|
+
monit_shell_out!('restart')
|
152
|
+
end
|
153
|
+
|
154
|
+
def find_monit_status
|
155
|
+
re = /^Process '#{new_resource.service_name}'\s+status\s+(\w.+)$/
|
156
|
+
status_cmd = monit_shell_out!('status') do |cmd|
|
157
|
+
# Command fails if it has an error, does't have Process line, or
|
158
|
+
# does have Initializing.
|
159
|
+
cmd.error? || cmd.stdout !~ re || cmd.stdout =~ /Initializing/
|
160
|
+
end
|
161
|
+
status_cmd.stdout =~ re && $1
|
162
|
+
end
|
163
|
+
|
164
|
+
def monit_shell_out!(monit_cmd, timeout: DEFAULT_TIMEOUT, wait: DEFAULT_WAIT, &block)
|
165
|
+
while true
|
166
|
+
cmd_args = [new_resource.parent.monit_binary, '-c', new_resource.parent.config_path, monit_cmd, new_resource.service_name]
|
167
|
+
Chef::Log.debug("[#{new_resource}] Running #{cmd_args.join(' ')}")
|
168
|
+
cmd = poise_shell_out(cmd_args, user: new_resource.parent.owner, group: new_resource.parent.group)
|
169
|
+
error = block ? block.call(cmd) : cmd.error?
|
170
|
+
# If there was an error (or error-like output), sleep and try again.
|
171
|
+
if error
|
172
|
+
# We fell off the end of the timeout, doneburger.
|
173
|
+
if timeout <= 0
|
174
|
+
Chef::Log.debug("[#{new_resource}] Timeout while running `monit #{monit_cmd}`")
|
175
|
+
# If there was a run error, raise that first.
|
176
|
+
cmd.error!
|
177
|
+
# Otherwise we just didn't have the requested output, which is fine.
|
178
|
+
break
|
179
|
+
end
|
180
|
+
# Wait and try again.
|
181
|
+
Chef::Log.debug("[#{new_resource}] Failure running `monit #{monit_cmd}`, retrying in #{wait}")
|
182
|
+
timeout -= Kernel.sleep(wait)
|
183
|
+
else
|
184
|
+
# All's quiet on the western front.
|
185
|
+
break
|
186
|
+
end
|
187
|
+
end
|
188
|
+
cmd
|
189
|
+
end
|
190
|
+
|
191
|
+
end
|
192
|
+
end
|
193
|
+
end
|
194
|
+
end
|
@@ -0,0 +1,127 @@
|
|
1
|
+
#
|
2
|
+
# Copyright 2015, Noah Kantrowitz
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'chef/resource'
|
18
|
+
require 'chef/provider'
|
19
|
+
require 'poise'
|
20
|
+
|
21
|
+
require 'poise_service/resources/poise_service_test'
|
22
|
+
|
23
|
+
|
24
|
+
module PoiseMonit
|
25
|
+
module Resources
|
26
|
+
# (see PoiseMonitTest::Resource)
|
27
|
+
module PoiseMonitTest
|
28
|
+
# A `monit_test` resource for integration testing monit providers.
|
29
|
+
# This is used in Test-Kitchen tests to ensure all providers behave
|
30
|
+
# similarly.
|
31
|
+
#
|
32
|
+
# @since 1.0.0
|
33
|
+
# @provides monit_test
|
34
|
+
# @action run
|
35
|
+
# @example
|
36
|
+
# monit_test 'system' do
|
37
|
+
# monit_provider :system
|
38
|
+
# base_port 5000
|
39
|
+
# end
|
40
|
+
class Resource < Chef::Resource
|
41
|
+
include Poise
|
42
|
+
provides(:monit_test)
|
43
|
+
actions(:run)
|
44
|
+
|
45
|
+
# @!attribute monit_provider
|
46
|
+
# Monit provider to set for the test group.
|
47
|
+
# @return [Symbol]
|
48
|
+
attribute(:monit_provider, kind_of: Symbol)
|
49
|
+
# @!attribute path
|
50
|
+
# Path for writing files for this test group.
|
51
|
+
# @return [String]
|
52
|
+
attribute(:path, kind_of: String, default: lazy { "/root/monit_test_#{name}" })
|
53
|
+
# @!attribute base_port
|
54
|
+
# Port number to start from for the test group.
|
55
|
+
# @return [Integer]
|
56
|
+
attribute(:base_port, kind_of: Integer)
|
57
|
+
end
|
58
|
+
|
59
|
+
# Provider for `monit_test`.
|
60
|
+
#
|
61
|
+
# @see Resource
|
62
|
+
# @provides monit_test
|
63
|
+
class Provider < Chef::Provider
|
64
|
+
include Poise
|
65
|
+
provides(:monit_test)
|
66
|
+
|
67
|
+
# `run` action for `poise_service_test`. Create all test services.
|
68
|
+
#
|
69
|
+
# @return [void]
|
70
|
+
def action_run
|
71
|
+
notifying_block do
|
72
|
+
# Make the test output root.
|
73
|
+
directory new_resource.path
|
74
|
+
|
75
|
+
# Install Monit.
|
76
|
+
r = monit new_resource.name do
|
77
|
+
provider new_resource.monit_provider if new_resource.monit_provider
|
78
|
+
end
|
79
|
+
|
80
|
+
# Write out some config files.
|
81
|
+
monit_config 'file_test' do
|
82
|
+
content <<-EOH
|
83
|
+
CHECK FILE file_test PATH #{new_resource.path}/check
|
84
|
+
start = "/bin/touch #{new_resource.path}/check"
|
85
|
+
EOH
|
86
|
+
parent r
|
87
|
+
end
|
88
|
+
monit_service 'file_test' do
|
89
|
+
action :enable
|
90
|
+
parent r
|
91
|
+
end
|
92
|
+
file "#{new_resource.path}/service" do
|
93
|
+
content <<-EOH
|
94
|
+
#!/bin/bash
|
95
|
+
nohup /bin/bash -c 'echo $$ >> #{new_resource.path}/pid; while sleep 1; do true; done' &
|
96
|
+
EOH
|
97
|
+
mode '700'
|
98
|
+
end
|
99
|
+
monit_config 'process_test' do
|
100
|
+
content <<-EOH
|
101
|
+
check process process_test with pidfile #{new_resource.path}/pid
|
102
|
+
start program = "#{new_resource.path}/service"
|
103
|
+
EOH
|
104
|
+
parent r
|
105
|
+
end
|
106
|
+
monit_service "process_test" do
|
107
|
+
action [:enable, :start]
|
108
|
+
parent r
|
109
|
+
end
|
110
|
+
|
111
|
+
# Run some monit commands.
|
112
|
+
execute "#{r.monit_binary} -V -c '#{r.config_path}' > #{new_resource.path}/version"
|
113
|
+
execute "#{r.monit_binary} status -c '#{r.config_path}' > #{new_resource.path}/status"
|
114
|
+
|
115
|
+
# Run poise_service_test for the service provider.
|
116
|
+
poise_service_test "monit_#{new_resource.name}" do
|
117
|
+
base_port new_resource.base_port
|
118
|
+
service_provider :monit
|
119
|
+
service_options parent: r
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
#
|
2
|
+
# Copyright 2015, Noah Kantrowitz
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'poise_monit/service_providers/monit'
|
18
|
+
|
19
|
+
|
20
|
+
module PoiseMonit
|
21
|
+
# Providers for `poise_service`.
|
22
|
+
#
|
23
|
+
# @since 1.0.0
|
24
|
+
module ServiceProviders
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,124 @@
|
|
1
|
+
#
|
2
|
+
# Copyright 2015, Noah Kantrowitz
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'poise_service/service_providers/sysvinit'
|
18
|
+
|
19
|
+
|
20
|
+
module PoiseMonit
|
21
|
+
module ServiceProviders
|
22
|
+
# A `monit` provider for `poise_service`. This uses the sysvinit code to
|
23
|
+
# generate the underlying service script, but Monit to manage the service
|
24
|
+
# runtime.
|
25
|
+
#
|
26
|
+
# @since 1.0.0
|
27
|
+
# @provides monit
|
28
|
+
class Monit < PoiseService::ServiceProviders::Sysvinit
|
29
|
+
provides(:monit)
|
30
|
+
|
31
|
+
# Override the default reload action because monit_service doesn't
|
32
|
+
# support reload itself.
|
33
|
+
def action_reload
|
34
|
+
return if options['never_reload']
|
35
|
+
if running?
|
36
|
+
converge_by("reload service #{new_resource}") do
|
37
|
+
Process.kill(new_resource.reload_signal, pid)
|
38
|
+
Chef::Log.info("#{new_resource} reloaded")
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
private
|
44
|
+
|
45
|
+
def service_resource
|
46
|
+
@service_resource ||= PoiseMonit::Resources::MonitService::Resource.new(new_resource.service_name, run_context).tap do |r|
|
47
|
+
# Set standard resource parameters
|
48
|
+
r.enclosing_provider = self
|
49
|
+
r.source_line = new_resource.source_line
|
50
|
+
# Make sure we have a parent.
|
51
|
+
if options['parent']
|
52
|
+
r.parent options['parent']
|
53
|
+
else
|
54
|
+
begin
|
55
|
+
# Try to find a default parent, trigger an exception if not.
|
56
|
+
r.parent
|
57
|
+
rescue Poise::Error
|
58
|
+
# Use the default recipe to give us a parent the next time we ask.
|
59
|
+
include_recipe(node['poise-monit']['default_recipe'])
|
60
|
+
end
|
61
|
+
end
|
62
|
+
# Set some params on the service resource.
|
63
|
+
r.init_command(script_path)
|
64
|
+
# Mild encapulsation break, this is an internal detail of monit_config. :-/
|
65
|
+
r.monit_config_path(::File.join(r.parent.confd_path, "#{new_resource.service_name}.conf"))
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
def running?
|
70
|
+
begin
|
71
|
+
# Check if the PID is running.
|
72
|
+
pid && Process.kill(0, pid)
|
73
|
+
rescue Errno::ESRCH
|
74
|
+
false
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
# Patch Monit behavior in to service creation.
|
79
|
+
def create_service
|
80
|
+
super
|
81
|
+
create_monit_config
|
82
|
+
end
|
83
|
+
|
84
|
+
# Create the Monit configuration file.
|
85
|
+
def create_monit_config
|
86
|
+
# Scope closureeeeeee.
|
87
|
+
_options = options
|
88
|
+
_pid_file = pid_file
|
89
|
+
_parent = service_resource.parent
|
90
|
+
_script_path = script_path
|
91
|
+
monit_config new_resource.service_name do
|
92
|
+
cookbook 'poise-monit'
|
93
|
+
parent _parent
|
94
|
+
source 'monit_service.conf.erb'
|
95
|
+
variables service_resource: new_resource, options: _options, pid_file: _pid_file, script_path: _script_path
|
96
|
+
# Don't trigger a restart if the template doesn't already exist, this
|
97
|
+
# prevents restarting on the run that first creates the service.
|
98
|
+
restart_on_update = _options.fetch('restart_on_update', new_resource.restart_on_update)
|
99
|
+
if restart_on_update && ::File.exist?(path) # Path here is accessing MonitConfig::Resource#path.
|
100
|
+
mode = restart_on_update.to_s == 'immediately' ? :immediately : :delayed
|
101
|
+
notifies :restart, new_resource, mode
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
# Patch Monit behavior in to service teardown.
|
107
|
+
def destroy_service
|
108
|
+
delete_monit_config
|
109
|
+
super
|
110
|
+
end
|
111
|
+
|
112
|
+
# Delete the Monit configuration file.
|
113
|
+
def delete_monit_config
|
114
|
+
_parent = service_resource.parent
|
115
|
+
monit_config new_resource.service_name do
|
116
|
+
action :delete
|
117
|
+
parent _parent
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
end
|
122
|
+
end
|
123
|
+
end
|
124
|
+
|