eye 0.5.2 → 0.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +13 -5
- data/.travis.yml +1 -6
- data/CHANGES.md +12 -0
- data/README.md +5 -0
- data/Rakefile +4 -4
- data/bin/loader_eye +14 -3
- data/bin/runner +16 -0
- data/examples/dependency.eye +17 -0
- data/examples/plugin/README.md +15 -0
- data/examples/plugin/main.eye +15 -0
- data/examples/plugin/plugin.rb +63 -0
- data/examples/unicorn.eye +1 -1
- data/eye.gemspec +1 -2
- data/lib/eye.rb +1 -1
- data/lib/eye/checker.rb +16 -4
- data/lib/eye/checker/children_count.rb +44 -0
- data/lib/eye/checker/children_memory.rb +12 -0
- data/lib/eye/checker/socket.rb +9 -2
- data/lib/eye/child_process.rb +6 -2
- data/lib/eye/cli.rb +13 -2
- data/lib/eye/cli/commands.rb +2 -2
- data/lib/eye/cli/server.rb +11 -3
- data/lib/eye/config.rb +2 -2
- data/lib/eye/controller/commands.rb +29 -2
- data/lib/eye/controller/helpers.rb +31 -6
- data/lib/eye/controller/load.rb +5 -6
- data/lib/eye/controller/options.rb +1 -1
- data/lib/eye/controller/send_command.rb +0 -1
- data/lib/eye/dsl.rb +2 -1
- data/lib/eye/dsl/application_opts.rb +4 -7
- data/lib/eye/dsl/group_opts.rb +2 -1
- data/lib/eye/dsl/helpers.rb +9 -1
- data/lib/eye/dsl/main.rb +11 -5
- data/lib/eye/dsl/opts.rb +5 -22
- data/lib/eye/dsl/process_opts.rb +20 -2
- data/lib/eye/dsl/pure_opts.rb +1 -1
- data/lib/eye/dsl/validation.rb +17 -2
- data/lib/eye/local.rb +79 -50
- data/lib/eye/notify.rb +5 -3
- data/lib/eye/notify/mail.rb +6 -2
- data/lib/eye/process.rb +3 -1
- data/lib/eye/process/children.rb +1 -1
- data/lib/eye/process/commands.rb +17 -6
- data/lib/eye/process/config.rb +6 -1
- data/lib/eye/process/data.rb +20 -0
- data/lib/eye/process/monitor.rb +10 -4
- data/lib/eye/process/states.rb +5 -2
- data/lib/eye/process/states_history.rb +1 -1
- data/lib/eye/process/system.rb +6 -2
- data/lib/eye/process/trigger.rb +0 -1
- data/lib/eye/process/validate.rb +8 -6
- data/lib/eye/process/watchers.rb +1 -7
- data/lib/eye/system.rb +14 -11
- data/lib/eye/system_resources.rb +8 -0
- data/lib/eye/trigger.rb +12 -4
- data/lib/eye/trigger/check_dependency.rb +30 -0
- data/lib/eye/trigger/stop_children.rb +4 -1
- data/lib/eye/trigger/wait_dependency.rb +49 -0
- data/lib/eye/utils.rb +13 -0
- metadata +41 -45
data/lib/eye/cli.rb
CHANGED
@@ -58,7 +58,7 @@ class Eye::Cli < Thor
|
|
58
58
|
|
59
59
|
if options[:foreground]
|
60
60
|
# in foreground we stop another server, and run just 1 current config version
|
61
|
-
error!("foreground expected only one config") if configs.size
|
61
|
+
error!("foreground expected only one config") if configs.size > 1
|
62
62
|
server_start_foreground(configs.first)
|
63
63
|
|
64
64
|
elsif server_started?
|
@@ -71,7 +71,15 @@ class Eye::Cli < Thor
|
|
71
71
|
end
|
72
72
|
|
73
73
|
desc "quit", "eye-daemon quit"
|
74
|
+
method_option :stop_all, :type => :boolean, :aliases => "-s"
|
75
|
+
method_option :timeout, :type => :string, :aliases => "-t", :default => "600"
|
74
76
|
def quit
|
77
|
+
if options[:stop_all]
|
78
|
+
Eye::Local.client_timeout = options[:timeout].to_i
|
79
|
+
cmd(:stop_all, options[:timeout].to_i)
|
80
|
+
end
|
81
|
+
|
82
|
+
Eye::Local.client_timeout = 5
|
75
83
|
res = _cmd(:quit)
|
76
84
|
|
77
85
|
# if eye server got crazy, stop by force
|
@@ -80,7 +88,7 @@ class Eye::Cli < Thor
|
|
80
88
|
# remove pid_file
|
81
89
|
File.delete(Eye::Local.pid_path) if File.exists?(Eye::Local.pid_path)
|
82
90
|
|
83
|
-
say "
|
91
|
+
say "Quit :(", :yellow
|
84
92
|
end
|
85
93
|
|
86
94
|
[:start, :stop, :restart, :unmonitor, :monitor, :delete, :match].each do |_cmd|
|
@@ -171,4 +179,7 @@ private
|
|
171
179
|
end
|
172
180
|
end
|
173
181
|
|
182
|
+
def self.exit_on_failure?
|
183
|
+
true
|
184
|
+
end
|
174
185
|
end
|
data/lib/eye/cli/commands.rb
CHANGED
@@ -42,9 +42,9 @@ private
|
|
42
42
|
res[:backtrace].to_a.each{|line| say line, :red }
|
43
43
|
else
|
44
44
|
if opts[:syntax]
|
45
|
-
say '
|
45
|
+
say 'Config ok!', :green if !res[:empty]
|
46
46
|
else
|
47
|
-
say '
|
47
|
+
say 'Config loaded!', :green if !res[:empty]
|
48
48
|
end
|
49
49
|
|
50
50
|
if opts[:print_config]
|
data/lib/eye/cli/server.rb
CHANGED
@@ -30,8 +30,13 @@ private
|
|
30
30
|
end
|
31
31
|
|
32
32
|
args = []
|
33
|
-
args += ['
|
34
|
-
args += ['
|
33
|
+
args += ['--config', conf] if conf
|
34
|
+
args += ['--logger', 'stdout']
|
35
|
+
if Eye::Local.local_runner
|
36
|
+
args += ['--stop_all']
|
37
|
+
args += ['--dir', Eye::Local.dir]
|
38
|
+
args += ['--config', Eye::Local.eyefile] unless conf
|
39
|
+
end
|
35
40
|
|
36
41
|
Process.exec(ruby_path, loader_path, *args)
|
37
42
|
end
|
@@ -43,6 +48,8 @@ private
|
|
43
48
|
ensure_stop_previous_server
|
44
49
|
|
45
50
|
args = []
|
51
|
+
args += ['--dir', Eye::Local.dir] if Eye::Local.local_runner
|
52
|
+
|
46
53
|
opts = {:out => '/dev/null', :err => '/dev/null', :in => '/dev/null',
|
47
54
|
:chdir => '/', :pgroup => true}
|
48
55
|
|
@@ -55,8 +62,9 @@ private
|
|
55
62
|
end
|
56
63
|
|
57
64
|
configs.unshift(Eye::Local.eyeconfig) if File.exists?(Eye::Local.eyeconfig)
|
65
|
+
configs << Eye::Local.eyefile if Eye::Local.local_runner
|
58
66
|
|
59
|
-
say '
|
67
|
+
say 'Eye started!', :green
|
60
68
|
|
61
69
|
if !configs.empty?
|
62
70
|
say_load_result cmd(:load, *configs)
|
data/lib/eye/config.rb
CHANGED
@@ -21,7 +21,7 @@ class Eye::Config
|
|
21
21
|
end
|
22
22
|
|
23
23
|
# raise an error if config wrong
|
24
|
-
def validate!
|
24
|
+
def validate!(localize = true)
|
25
25
|
all_processes = processes
|
26
26
|
|
27
27
|
# Check blank pid_files
|
@@ -55,7 +55,7 @@ class Eye::Config
|
|
55
55
|
|
56
56
|
# validate processes with their own validate
|
57
57
|
all_processes.each do |process_cfg|
|
58
|
-
Eye::Process.validate process_cfg
|
58
|
+
Eye::Process.validate process_cfg, localize
|
59
59
|
end
|
60
60
|
|
61
61
|
# just to be sure ENV was not removed
|
@@ -1,8 +1,14 @@
|
|
1
1
|
module Eye::Controller::Commands
|
2
2
|
|
3
|
+
NOT_IMPORTANT_COMMANDS = [:info_data, :short_data, :debug_data, :history_data, :ping,
|
4
|
+
:logger_dev, :match, :explain, :check]
|
5
|
+
|
3
6
|
# Main method, answer for the client command
|
4
7
|
def command(cmd, *args)
|
5
|
-
|
8
|
+
msg = "command: #{cmd} #{args * ', '}"
|
9
|
+
|
10
|
+
log_str = "=> #{msg}"
|
11
|
+
NOT_IMPORTANT_COMMANDS.include?(cmd) ? debug(log_str) : info(log_str)
|
6
12
|
|
7
13
|
start_at = Time.now
|
8
14
|
cmd = cmd.to_sym
|
@@ -18,6 +24,8 @@ module Eye::Controller::Commands
|
|
18
24
|
load(*args)
|
19
25
|
when :quit
|
20
26
|
quit
|
27
|
+
when :stop_all
|
28
|
+
stop_all(*args)
|
21
29
|
when :check
|
22
30
|
check(*args)
|
23
31
|
when :explain
|
@@ -44,7 +52,9 @@ module Eye::Controller::Commands
|
|
44
52
|
end
|
45
53
|
|
46
54
|
GC.start
|
47
|
-
|
55
|
+
|
56
|
+
log_str = "<= #{msg} (#{Time.now - start_at}s)"
|
57
|
+
NOT_IMPORTANT_COMMANDS.include?(cmd) ? debug(log_str) : info(log_str)
|
48
58
|
|
49
59
|
res
|
50
60
|
end
|
@@ -58,4 +68,21 @@ private
|
|
58
68
|
Eye::System.send_signal($$, :KILL)
|
59
69
|
end
|
60
70
|
|
71
|
+
# stop all processes and wait
|
72
|
+
def stop_all(timeout = nil)
|
73
|
+
exclusive do
|
74
|
+
send_command :break_chain, 'all'
|
75
|
+
send_command :stop, 'all'
|
76
|
+
end
|
77
|
+
|
78
|
+
# wait until all processes goes to unmonitored
|
79
|
+
timeout ||= 100
|
80
|
+
|
81
|
+
all_processes.pmap do |p|
|
82
|
+
p.wait_for_condition(timeout, 0.3) do
|
83
|
+
p.state_name == :unmonitored
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
61
88
|
end
|
@@ -2,8 +2,9 @@ module Eye::Controller::Helpers
|
|
2
2
|
|
3
3
|
def set_proc_line
|
4
4
|
str = Eye::PROCLINE
|
5
|
-
str += "
|
6
|
-
str += "
|
5
|
+
str += " [#{@applications.map(&:name) * ', '}]" if @applications.present?
|
6
|
+
str += " (v #{ENV['EYE_V']})" if ENV['EYE_V']
|
7
|
+
str += " (in #{Eye::Local.dir})" if Eye::Local.local_runner
|
7
8
|
$0 = str
|
8
9
|
end
|
9
10
|
|
@@ -18,19 +19,43 @@ module Eye::Controller::Helpers
|
|
18
19
|
end
|
19
20
|
|
20
21
|
def process_by_name(name)
|
21
|
-
|
22
|
+
name = name.to_s
|
23
|
+
all_processes.detect { |c| c.name == name }
|
22
24
|
end
|
23
25
|
|
24
26
|
def process_by_full_name(name)
|
25
|
-
|
27
|
+
name = name.to_s
|
28
|
+
all_processes.detect { |c| c.full_name == name }
|
29
|
+
end
|
30
|
+
|
31
|
+
def find_nearest_process(name, group_name = nil, app_name = nil)
|
32
|
+
return process_by_full_name(name) if name.include?(':')
|
33
|
+
|
34
|
+
if app_name
|
35
|
+
app = application_by_name(app_name)
|
36
|
+
app.groups.each do |gr|
|
37
|
+
p = gr.processes.detect { |c| c.name == name }
|
38
|
+
return p if p
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
if group_name
|
43
|
+
gr = group_by_name(group_name)
|
44
|
+
p = gr.processes.detect { |c| c.name == name }
|
45
|
+
return p if p
|
46
|
+
end
|
47
|
+
|
48
|
+
process_by_name(name)
|
26
49
|
end
|
27
50
|
|
28
51
|
def group_by_name(name)
|
29
|
-
|
52
|
+
name = name.to_s
|
53
|
+
all_groups.detect { |c| c.name == name }
|
30
54
|
end
|
31
55
|
|
32
56
|
def application_by_name(name)
|
33
|
-
|
57
|
+
name = name.to_s
|
58
|
+
@applications.detect { |c| c.name == name }
|
34
59
|
end
|
35
60
|
|
36
61
|
def all_processes
|
data/lib/eye/controller/load.rb
CHANGED
@@ -11,7 +11,7 @@ module Eye::Controller::Load
|
|
11
11
|
def load(*args)
|
12
12
|
h = args.extract_options!
|
13
13
|
obj_strs = args.flatten
|
14
|
-
info "loading: #{obj_strs}"
|
14
|
+
info "=> loading: #{obj_strs}"
|
15
15
|
|
16
16
|
res = Hash.new
|
17
17
|
|
@@ -24,9 +24,8 @@ module Eye::Controller::Load
|
|
24
24
|
end
|
25
25
|
|
26
26
|
set_proc_line
|
27
|
-
save_cache
|
28
27
|
|
29
|
-
info "
|
28
|
+
info "<= loading: #{obj_strs}, in: <#{$$}>"
|
30
29
|
|
31
30
|
res
|
32
31
|
end
|
@@ -34,7 +33,7 @@ module Eye::Controller::Load
|
|
34
33
|
private
|
35
34
|
|
36
35
|
# regexp for clean backtrace to show for user
|
37
|
-
BT_REGX = %r[/lib/eye/|lib/celluloid|internal:prelude|logger.rb:|active_support/core_ext|shellwords.rb].freeze
|
36
|
+
BT_REGX = %r[/lib/eye/|lib/celluloid|internal:prelude|logger.rb:|active_support/core_ext|shellwords.rb|kernel/bootstrap].freeze
|
38
37
|
|
39
38
|
def catch_load_error(filename = nil, &block)
|
40
39
|
{ :error => false, :config => yield }
|
@@ -46,7 +45,7 @@ private
|
|
46
45
|
|
47
46
|
# filter backtrace for user output
|
48
47
|
bt = (ex.backtrace || [])
|
49
|
-
bt = bt.reject{|line| line.to_s =~ BT_REGX }
|
48
|
+
bt = bt.reject{|line| line.to_s =~ BT_REGX } unless ENV['EYE_FULL_BACKTRACE']
|
50
49
|
error bt.join("\n")
|
51
50
|
|
52
51
|
res = { :error => true, :message => ex.message }
|
@@ -84,7 +83,7 @@ private
|
|
84
83
|
debug "parsing: #{filename}"
|
85
84
|
|
86
85
|
cfg = Eye::Dsl.parse(nil, filename)
|
87
|
-
@current_config.merge(cfg).validate! # just validate summary config here
|
86
|
+
@current_config.merge(cfg).validate!(false) # just validate summary config here
|
88
87
|
Eye.parsed_config = nil # remove link on config, for better gc
|
89
88
|
cfg
|
90
89
|
end
|
data/lib/eye/dsl.rb
CHANGED
@@ -27,6 +27,7 @@ class Eye::Dsl
|
|
27
27
|
def parse(content = nil, filename = nil)
|
28
28
|
Eye.parsed_config = Eye::Config.new
|
29
29
|
Eye.parsed_filename = filename
|
30
|
+
Eye.parsed_default_app = nil
|
30
31
|
|
31
32
|
content = File.read(filename) if content.blank?
|
32
33
|
|
@@ -34,7 +35,7 @@ class Eye::Dsl
|
|
34
35
|
Kernel.eval(content, Eye::BINDING, filename.to_s)
|
35
36
|
end
|
36
37
|
|
37
|
-
Eye.parsed_config.validate!
|
38
|
+
Eye.parsed_config.validate!(false)
|
38
39
|
Eye.parsed_config
|
39
40
|
end
|
40
41
|
|
@@ -17,14 +17,11 @@ class Eye::Dsl::ApplicationOpts < Eye::Dsl::Opts
|
|
17
17
|
opts = Eye::Dsl::GroupOpts.new(name, self)
|
18
18
|
opts.instance_eval(&block)
|
19
19
|
|
20
|
-
|
21
|
-
|
20
|
+
@config[:groups] ||= {}
|
21
|
+
@config[:groups][name.to_s] ||= {}
|
22
22
|
|
23
|
-
|
24
|
-
@config[:groups][name.to_s]
|
25
|
-
@config[:groups][name.to_s].merge!(cfg)
|
26
|
-
@config[:groups][name.to_s][:processes] ||= {}
|
27
|
-
@config[:groups][name.to_s][:processes].merge!(processes)
|
23
|
+
if cfg = opts.config
|
24
|
+
Eye::Utils.deep_merge!(@config[:groups][name.to_s], cfg)
|
28
25
|
end
|
29
26
|
|
30
27
|
Eye::Dsl.debug "<= group #{name}"
|
data/lib/eye/dsl/group_opts.rb
CHANGED
@@ -18,7 +18,8 @@ class Eye::Dsl::GroupOpts < Eye::Dsl::Opts
|
|
18
18
|
opts = Eye::Dsl::ProcessOpts.new(name, self)
|
19
19
|
opts.instance_eval(&block)
|
20
20
|
@config[:processes] ||= {}
|
21
|
-
@config[:processes][name.to_s]
|
21
|
+
@config[:processes][name.to_s] ||= {}
|
22
|
+
Eye::Utils.deep_merge!(@config[:processes][name.to_s], opts.config) if opts.config
|
22
23
|
|
23
24
|
Eye::Dsl.debug "<= process #{name}"
|
24
25
|
opts
|
data/lib/eye/dsl/helpers.rb
CHANGED
data/lib/eye/dsl/main.rb
CHANGED
@@ -1,14 +1,20 @@
|
|
1
1
|
module Eye::Dsl::Main
|
2
|
-
attr_accessor :parsed_config, :parsed_filename
|
2
|
+
attr_accessor :parsed_config, :parsed_filename, :parsed_default_app
|
3
3
|
|
4
4
|
def application(name, &block)
|
5
5
|
Eye::Dsl.check_name(name)
|
6
|
+
name = name.to_s
|
6
7
|
|
7
8
|
Eye::Dsl.debug "=> app: #{name}"
|
8
|
-
opts = Eye::Dsl::ApplicationOpts.new(name)
|
9
|
-
opts.instance_eval(&block)
|
10
9
|
|
11
|
-
|
10
|
+
if name == '__default__'
|
11
|
+
@parsed_default_app ||= Eye::Dsl::ApplicationOpts.new(name)
|
12
|
+
@parsed_default_app.instance_eval(&block)
|
13
|
+
else
|
14
|
+
opts = Eye::Dsl::ApplicationOpts.new(name, @parsed_default_app)
|
15
|
+
opts.instance_eval(&block)
|
16
|
+
@parsed_config.applications[name] = opts.config if opts.config
|
17
|
+
end
|
12
18
|
|
13
19
|
Eye::Dsl.debug "<= app: #{name}"
|
14
20
|
end
|
@@ -35,7 +41,7 @@ module Eye::Dsl::Main
|
|
35
41
|
|
36
42
|
opts = Eye::Dsl::ConfigOpts.new
|
37
43
|
opts.instance_eval(&block)
|
38
|
-
@parsed_config.settings
|
44
|
+
Eye::Utils.deep_merge!(@parsed_config.settings, opts.config)
|
39
45
|
|
40
46
|
Eye::Dsl.debug '<= config'
|
41
47
|
end
|
data/lib/eye/dsl/opts.rb
CHANGED
@@ -4,11 +4,12 @@ class Eye::Dsl::Opts < Eye::Dsl::PureOpts
|
|
4
4
|
:stop_command, :restart_command, :uid, :gid ]
|
5
5
|
create_options_methods(STR_OPTIONS, String)
|
6
6
|
|
7
|
-
BOOL_OPTIONS = [ :daemonize, :keep_alive, :auto_start, :stop_on_delete, :clear_pid ]
|
7
|
+
BOOL_OPTIONS = [ :daemonize, :keep_alive, :auto_start, :stop_on_delete, :clear_pid, :preserve_fds, :use_leaf_child ]
|
8
8
|
create_options_methods(BOOL_OPTIONS, [TrueClass, FalseClass])
|
9
9
|
|
10
10
|
INTERVAL_OPTIONS = [ :check_alive_period, :start_timeout, :restart_timeout, :stop_timeout, :start_grace,
|
11
|
-
:restart_grace, :stop_grace, :children_update_period, :restore_in
|
11
|
+
:restart_grace, :stop_grace, :children_update_period, :restore_in,
|
12
|
+
:auto_update_pidfile_grace, :revert_fuckup_pidfile_grace ]
|
12
13
|
create_options_methods(INTERVAL_OPTIONS, [Fixnum, Float])
|
13
14
|
|
14
15
|
create_options_methods([:environment], Hash)
|
@@ -23,7 +24,7 @@ class Eye::Dsl::Opts < Eye::Dsl::PureOpts
|
|
23
24
|
@config[:group] = parent.name if parent.is_a?(Eye::Dsl::GroupOpts)
|
24
25
|
|
25
26
|
# hack for full name
|
26
|
-
@full_name = parent.full_name if @name == '__default__'
|
27
|
+
@full_name = parent.full_name if @name == '__default__' && parent.respond_to?(:full_name)
|
27
28
|
end
|
28
29
|
|
29
30
|
def checks(type, opts = {})
|
@@ -118,25 +119,7 @@ class Eye::Dsl::Opts < Eye::Dsl::PureOpts
|
|
118
119
|
def scoped(&block)
|
119
120
|
h = self.class.new(self.name, self)
|
120
121
|
h.instance_eval(&block)
|
121
|
-
|
122
|
-
groups = h.config.delete :groups
|
123
|
-
|
124
|
-
if groups.present?
|
125
|
-
config[:groups] ||= {}
|
126
|
-
groups.each do |name, cfg|
|
127
|
-
processes = cfg.delete(:processes) || {}
|
128
|
-
config[:groups][name] ||= {}
|
129
|
-
config[:groups][name].merge!(cfg)
|
130
|
-
config[:groups][name][:processes] ||= {}
|
131
|
-
config[:groups][name][:processes].merge!(processes)
|
132
|
-
end
|
133
|
-
end
|
134
|
-
|
135
|
-
processes = h.config.delete :processes
|
136
|
-
if processes.present?
|
137
|
-
config[:processes] ||= {}
|
138
|
-
config[:processes].merge!(processes)
|
139
|
-
end
|
122
|
+
Eye::Utils.deep_merge!(config, h.config, [:groups, :processes])
|
140
123
|
end
|
141
124
|
|
142
125
|
# execute part of config on particular server
|
data/lib/eye/dsl/process_opts.rb
CHANGED
@@ -4,7 +4,7 @@ class Eye::Dsl::ProcessOpts < Eye::Dsl::Opts
|
|
4
4
|
opts = Eye::Dsl::ChildProcessOpts.new
|
5
5
|
opts.instance_eval(&block) if block
|
6
6
|
@config[:monitor_children] ||= {}
|
7
|
-
@config[:monitor_children]
|
7
|
+
Eye::Utils.deep_merge!(@config[:monitor_children], opts.config)
|
8
8
|
end
|
9
9
|
|
10
10
|
alias xmonitor_children nop
|
@@ -15,4 +15,22 @@ class Eye::Dsl::ProcessOpts < Eye::Dsl::Opts
|
|
15
15
|
alias app application
|
16
16
|
alias group parent
|
17
17
|
|
18
|
-
|
18
|
+
def depend_on(names, opts = {})
|
19
|
+
names = Array(names).map(&:to_s)
|
20
|
+
trigger("wait_dependency_#{unique_num}", {:names => names}.merge(opts))
|
21
|
+
nm = @config[:name]
|
22
|
+
names.each do |name|
|
23
|
+
parent.process(name) do
|
24
|
+
trigger("check_dependency_#{unique_num}", :names => [ nm ] )
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
private
|
30
|
+
|
31
|
+
def unique_num
|
32
|
+
$unique_num ||= 0
|
33
|
+
$unique_num += 1
|
34
|
+
end
|
35
|
+
|
36
|
+
end
|