newrelic_rpm 3.3.5 → 3.4.0.beta1
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of newrelic_rpm might be problematic. Click here for more details.
- data/CHANGELOG +8 -5
- data/lib/new_relic/agent.rb +11 -3
- data/lib/new_relic/agent/agent.rb +68 -223
- data/lib/new_relic/agent/error_collector.rb +1 -1
- data/lib/new_relic/agent/instrumentation/resque.rb +80 -0
- data/lib/new_relic/agent/instrumentation/sinatra.rb +2 -0
- data/lib/new_relic/agent/new_relic_service.rb +221 -0
- data/lib/new_relic/agent/pipe_channel_manager.rb +151 -0
- data/lib/new_relic/agent/pipe_service.rb +57 -0
- data/lib/new_relic/agent/samplers/delayed_job_sampler.rb +4 -34
- data/lib/new_relic/agent/sql_sampler.rb +1 -5
- data/lib/new_relic/agent/stats_engine/transactions.rb +2 -2
- data/lib/new_relic/agent/transaction_sampler.rb +0 -2
- data/lib/new_relic/control/configuration.rb +1 -1
- data/lib/new_relic/control/instance_methods.rb +2 -1
- data/lib/new_relic/language_support.rb +26 -0
- data/lib/new_relic/transaction_sample.rb +2 -6
- data/lib/new_relic/version.rb +3 -3
- data/newrelic_rpm.gemspec +16 -5
- data/test/new_relic/agent/agent/connect_test.rb +39 -23
- data/test/new_relic/agent/agent_test.rb +25 -4
- data/test/new_relic/agent/database_test.rb +12 -0
- data/test/new_relic/agent/new_relic_service_test.rb +151 -0
- data/test/new_relic/agent/pipe_channel_manager_test.rb +114 -0
- data/test/new_relic/agent/pipe_service_test.rb +113 -0
- data/test/new_relic/agent/rpm_agent_test.rb +3 -30
- data/test/new_relic/agent/transaction_sample_builder_test.rb +0 -1
- data/test/new_relic/agent/transaction_sampler_test.rb +6 -6
- data/test/new_relic/agent/worker_loop_test.rb +2 -2
- data/test/new_relic/agent_test.rb +73 -3
- data/test/new_relic/control/configuration_test.rb +0 -7
- data/test/new_relic/control_test.rb +3 -3
- data/test/new_relic/delayed_job_injection_test.rb +6 -1
- data/test/new_relic/fake_collector.rb +210 -0
- data/test/new_relic/fake_service.rb +44 -0
- data/test/script/ci.sh +6 -6
- data/test/script/ci_agent-tests_runner.sh +82 -0
- data/test/script/ci_multiverse_runner.sh +35 -0
- data/test/test_contexts.rb +1 -0
- data/test/test_helper.rb +3 -1
- metadata +359 -324
@@ -754,19 +754,19 @@ class NewRelic::Agent::TransactionSamplerTest < Test::Unit::TestCase
|
|
754
754
|
run_sample_trace
|
755
755
|
|
756
756
|
slowest = @sampler.harvest(nil, 0)[0]
|
757
|
-
|
758
|
-
|
759
|
-
|
760
|
-
|
757
|
+
first_duration = slowest.duration
|
758
|
+
assert((first_duration >= 0.1),
|
759
|
+
"expected sample duration >= 0.1, but was: #{slowest.duration.inspect}")
|
761
760
|
|
762
761
|
run_sample_trace { sleep 0.0001 }
|
763
762
|
not_as_slow = @sampler.harvest(slowest, 0)[0]
|
764
763
|
assert((not_as_slow == slowest), "Should re-harvest the same transaction since it should be slower than the new transaction - expected #{slowest.inspect} but got #{not_as_slow.inspect}")
|
765
764
|
|
766
|
-
run_sample_trace { sleep 0.
|
765
|
+
run_sample_trace { sleep(first_duration + 0.1) }
|
766
|
+
|
767
767
|
new_slowest = @sampler.harvest(slowest, 0)[0]
|
768
768
|
assert((new_slowest != slowest), "Should not harvest the same trace since the new one should be slower")
|
769
|
-
assert((new_slowest.duration >= 0.
|
769
|
+
assert((new_slowest.duration >= first_duration + 0.1), "Slowest duration must be >= #{first_duration + 0.1}, but was: #{new_slowest.duration.inspect}")
|
770
770
|
end
|
771
771
|
|
772
772
|
def test_prepare_to_send
|
@@ -23,7 +23,7 @@ class NewRelic::Agent::WorkerLoopTest < Test::Unit::TestCase
|
|
23
23
|
# This shows how the tasks stay aligned with the period and don't drift.
|
24
24
|
count = 0
|
25
25
|
start = Time.now
|
26
|
-
@worker_loop.run(0.
|
26
|
+
@worker_loop.run(0.03) do
|
27
27
|
count +=1
|
28
28
|
if count == 3
|
29
29
|
@worker_loop.stop
|
@@ -31,7 +31,7 @@ class NewRelic::Agent::WorkerLoopTest < Test::Unit::TestCase
|
|
31
31
|
end
|
32
32
|
end
|
33
33
|
elapsed = Time.now - start
|
34
|
-
assert_in_delta 0.
|
34
|
+
assert_in_delta 0.09, elapsed, 0.03
|
35
35
|
end
|
36
36
|
def test_task_error__standard
|
37
37
|
@logger.expects(:debug)
|
@@ -1,4 +1,6 @@
|
|
1
1
|
require File.expand_path(File.join(File.dirname(__FILE__),'..','test_helper'))
|
2
|
+
require 'ostruct'
|
3
|
+
|
2
4
|
module NewRelic
|
3
5
|
class MainAgentTest < Test::Unit::TestCase
|
4
6
|
|
@@ -22,6 +24,46 @@ module NewRelic
|
|
22
24
|
mock_agent.expects(:after_fork).with({})
|
23
25
|
NewRelic::Agent.after_fork
|
24
26
|
end
|
27
|
+
|
28
|
+
def test_after_fork_sets_forked_flag
|
29
|
+
agent = NewRelic::Agent::Agent.new
|
30
|
+
assert !agent.forked?
|
31
|
+
agent.after_fork
|
32
|
+
|
33
|
+
assert agent.forked?
|
34
|
+
end
|
35
|
+
|
36
|
+
|
37
|
+
if NewRelic::LanguageSupport.can_fork? &&
|
38
|
+
!NewRelic::LanguageSupport.using_version?('1.9.1')
|
39
|
+
def test_timeslice_harvest_with_after_fork_report_to_channel
|
40
|
+
NewRelic::Control.instance.stubs(:agent_enabled?).returns(true)
|
41
|
+
NewRelic::Control.instance.stubs(:monitor_mode?).returns(true)
|
42
|
+
|
43
|
+
NewRelic::Agent::Agent.instance.service = NewRelic::FakeService.new
|
44
|
+
NewRelic::Agent.shutdown # make sure the agent is not already started
|
45
|
+
NewRelic::Agent.manual_start(:license_key => ('1234567890' * 4),
|
46
|
+
:start_channel_listener => true)
|
47
|
+
|
48
|
+
metric = 'Custom/test/method'
|
49
|
+
NewRelic::Agent.instance.stats_engine.get_stats_no_scope(metric) \
|
50
|
+
.record_data_point(1.0)
|
51
|
+
|
52
|
+
NewRelic::Agent::PipeChannelManager.listener.close_all_pipes
|
53
|
+
NewRelic::Agent.register_report_channel(:agent_test) # before fork
|
54
|
+
pid = Process.fork do
|
55
|
+
NewRelic::Agent.after_fork(:report_to_channel => :agent_test)
|
56
|
+
NewRelic::Agent.agent.stats_engine.get_stats_no_scope(metric) \
|
57
|
+
.record_data_point(2.0)
|
58
|
+
end
|
59
|
+
Process.wait(pid)
|
60
|
+
NewRelic::Agent::PipeChannelManager.listener.stop
|
61
|
+
|
62
|
+
engine = NewRelic::Agent.agent.stats_engine
|
63
|
+
assert_equal(3.0, engine.lookup_stats(metric).total_call_time)
|
64
|
+
assert_equal(2, engine.lookup_stats(metric).call_count)
|
65
|
+
end
|
66
|
+
end
|
25
67
|
|
26
68
|
def test_reset_stats
|
27
69
|
mock_agent = mocked_agent
|
@@ -41,6 +83,15 @@ module NewRelic
|
|
41
83
|
NewRelic::Agent.manual_start(:sync_startup => false)
|
42
84
|
end
|
43
85
|
|
86
|
+
def test_manual_start_starts_channel_listener
|
87
|
+
NewRelic::Agent::PipeChannelManager.listener.stop
|
88
|
+
NewRelic::Agent.agent.service = NewRelic::FakeService.new
|
89
|
+
NewRelic::Agent.manual_start(:start_channel_listener => true)
|
90
|
+
assert NewRelic::Agent::PipeChannelManager.listener.started?
|
91
|
+
NewRelic::Agent::PipeChannelManager.listener.stop
|
92
|
+
NewRelic::Agent.shutdown
|
93
|
+
end
|
94
|
+
|
44
95
|
def test_logger
|
45
96
|
control = mocked_control
|
46
97
|
control.expects(:log)
|
@@ -169,7 +220,14 @@ module NewRelic
|
|
169
220
|
NewRelic::Agent.load_data
|
170
221
|
NewRelic::Control.instance['disable_serialization'] = false
|
171
222
|
end
|
172
|
-
|
223
|
+
|
224
|
+
def test_register_report_channel
|
225
|
+
NewRelic::Agent.register_report_channel(:channel_id)
|
226
|
+
assert NewRelic::Agent::PipeChannelManager.channels[:channel_id] \
|
227
|
+
.kind_of?(NewRelic::Agent::PipeChannelManager::Pipe)
|
228
|
+
NewRelic::Agent::PipeChannelManager.listener.close_all_pipes
|
229
|
+
end
|
230
|
+
|
173
231
|
private
|
174
232
|
|
175
233
|
def mocked_agent
|
@@ -177,9 +235,21 @@ module NewRelic
|
|
177
235
|
NewRelic::Agent.stubs(:agent).returns(agent)
|
178
236
|
agent
|
179
237
|
end
|
180
|
-
|
238
|
+
|
181
239
|
def mocked_control
|
182
|
-
|
240
|
+
server = NewRelic::Control::Server.new('localhost', 3000)
|
241
|
+
control = OpenStruct.new(:license_key => 'abcdef',
|
242
|
+
:server => server)
|
243
|
+
control.instance_eval do
|
244
|
+
def [](key)
|
245
|
+
nil
|
246
|
+
end
|
247
|
+
|
248
|
+
def fetch(k,d)
|
249
|
+
nil
|
250
|
+
end
|
251
|
+
end
|
252
|
+
|
183
253
|
NewRelic::Control.stubs(:instance).returns(control)
|
184
254
|
control
|
185
255
|
end
|
@@ -74,11 +74,4 @@ class NewRelic::Control::ConfigurationTest < Test::Unit::TestCase
|
|
74
74
|
DependencyDetection.send(:class_variable_set, '@@items', [])
|
75
75
|
assert NewRelic::Control.instance.disable_serialization?
|
76
76
|
end
|
77
|
-
|
78
|
-
def test_data_serialization_default_on_when_using_resque
|
79
|
-
DependencyDetection.defer { @name = :resque }
|
80
|
-
DependencyDetection.dependency_by_name(:resque).executed!
|
81
|
-
assert !NewRelic::Control.instance.disable_serialization?
|
82
|
-
DependencyDetection.send(:class_variable_set, '@@items', [])
|
83
|
-
end
|
84
77
|
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
require File.expand_path(File.join(File.dirname(__FILE__),'/../test_helper'))
|
2
|
-
class NewRelic::ControlTest < Test::Unit::TestCase
|
3
2
|
|
3
|
+
class NewRelic::ControlTest < Test::Unit::TestCase
|
4
4
|
attr_reader :control
|
5
5
|
|
6
6
|
def setup
|
@@ -244,10 +244,10 @@ class NewRelic::ControlTest < Test::Unit::TestCase
|
|
244
244
|
assert_equal 2.0, NewRelic::Control.instance['transaction_tracer']['explain_threshold']
|
245
245
|
assert_equal 'raw', NewRelic::Control.instance['transaction_tracer']['record_sql']
|
246
246
|
end
|
247
|
-
|
247
|
+
|
248
248
|
private
|
249
249
|
|
250
|
-
def forced_start
|
250
|
+
def forced_start(overrides={})
|
251
251
|
NewRelic::Agent.manual_start overrides
|
252
252
|
# This is to force the agent to start again.
|
253
253
|
NewRelic::Agent.instance.stubs(:started?).returns(nil)
|
@@ -5,8 +5,13 @@ class NewRelic::DelayedJobInstrumentationTest < Test::Unit::TestCase
|
|
5
5
|
Object.const_set('Delayed', Module.new) unless defined?(Delayed)
|
6
6
|
::Delayed.const_set('Worker', Class.new) unless defined?(::Delayed::Worker)
|
7
7
|
|
8
|
+
# on JRuby we need to make sure the worker isn't running, it might
|
9
|
+
# try to log
|
10
|
+
worker = NewRelic::Agent.agent.instance_variable_get(:@worker_loop)
|
11
|
+
worker.stop if worker
|
12
|
+
|
8
13
|
NewRelic::Agent.stubs(:logger).raises(NoMethodError,
|
9
|
-
|
14
|
+
'temporarily not allowed')
|
10
15
|
NewRelic::Agent.stubs(:respond_to?).with(:logger).returns(false)
|
11
16
|
|
12
17
|
assert DependencyDetection.detect!
|
@@ -0,0 +1,210 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'rack'
|
3
|
+
require 'uri'
|
4
|
+
require 'socket'
|
5
|
+
require 'timeout'
|
6
|
+
require 'ostruct'
|
7
|
+
|
8
|
+
module NewRelic
|
9
|
+
class FakeCollector
|
10
|
+
attr_accessor :agent_data, :mock
|
11
|
+
|
12
|
+
def initialize
|
13
|
+
@id_counter = 0
|
14
|
+
@base_expectations = {
|
15
|
+
'get_redirect_host' => 'localhost',
|
16
|
+
'connect' => { 'agent_run_id' => agent_run_id },
|
17
|
+
'metric_data' => { 'Some/Metric/Spec' => 1 },
|
18
|
+
'sql_trace_data' => nil,
|
19
|
+
'transaction_sample_data' => nil,
|
20
|
+
'error_data' => nil,
|
21
|
+
'shutdown' => nil,
|
22
|
+
}
|
23
|
+
reset
|
24
|
+
end
|
25
|
+
|
26
|
+
def agent_run_id
|
27
|
+
@id_counter += 1
|
28
|
+
end
|
29
|
+
|
30
|
+
def call(env)
|
31
|
+
req = ::Rack::Request.new(env)
|
32
|
+
res = ::Rack::Response.new
|
33
|
+
uri = URI.parse(req.url)
|
34
|
+
if uri.path =~ /agent_listener\/\d+\/.+\/(\w+)/
|
35
|
+
method = $1
|
36
|
+
if @mock.keys.include? method
|
37
|
+
res.write Marshal.dump(@mock[method])
|
38
|
+
else
|
39
|
+
res.status = 500
|
40
|
+
res.write "Method not found"
|
41
|
+
end
|
42
|
+
run_id = uri.query =~ /run_id=(\d+)/ ? $1 : nil
|
43
|
+
req.body.rewind
|
44
|
+
@agent_data << OpenStruct.new(:action => method,
|
45
|
+
:body => Marshal.load(req.body.read),
|
46
|
+
:run_id => run_id)
|
47
|
+
end
|
48
|
+
res.finish
|
49
|
+
end
|
50
|
+
|
51
|
+
def run(port=30303)
|
52
|
+
if is_port_available?('127.0.0.1', port)
|
53
|
+
@thread = Thread.new do
|
54
|
+
::Rack::Handler::WEBrick.run(self, :Port => port)
|
55
|
+
end
|
56
|
+
loop do
|
57
|
+
break if !is_port_available?('127.0.0.1', port)
|
58
|
+
sleep 0.01
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def stop
|
64
|
+
::Rack::Handler::WEBrick.shutdown
|
65
|
+
@thread.join
|
66
|
+
end
|
67
|
+
|
68
|
+
def reset
|
69
|
+
@mock = @base_expectations.dup
|
70
|
+
@id_counter = 0
|
71
|
+
@agent_data = []
|
72
|
+
end
|
73
|
+
|
74
|
+
def is_port_available?(ip, port)
|
75
|
+
begin
|
76
|
+
Timeout::timeout(1) do
|
77
|
+
begin
|
78
|
+
s = TCPSocket.new(ip, port)
|
79
|
+
s.close
|
80
|
+
return false
|
81
|
+
rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH
|
82
|
+
return true
|
83
|
+
end
|
84
|
+
end
|
85
|
+
rescue Timeout::Error
|
86
|
+
end
|
87
|
+
|
88
|
+
return true
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
if $0 == __FILE__
|
94
|
+
require 'test/unit'
|
95
|
+
require 'net/http'
|
96
|
+
|
97
|
+
class FakeCollectorTest < Test::Unit::TestCase
|
98
|
+
def setup
|
99
|
+
@collector = NewRelic::FakeCollector.new
|
100
|
+
@collector.run
|
101
|
+
end
|
102
|
+
|
103
|
+
def teardown
|
104
|
+
@collector.stop
|
105
|
+
end
|
106
|
+
|
107
|
+
def test_get_redirect
|
108
|
+
@collector.mock['get_redirect_host'] = 'test.example.com'
|
109
|
+
response = invoke('get_redirect_host')
|
110
|
+
|
111
|
+
assert_equal 'test.example.com', response
|
112
|
+
assert_equal 'get_redirect_host', @collector.agent_data[0].action
|
113
|
+
end
|
114
|
+
|
115
|
+
def test_connect
|
116
|
+
response = invoke('connect')
|
117
|
+
|
118
|
+
assert_equal 1, response['agent_run_id']
|
119
|
+
assert_equal 'connect', @collector.agent_data[0].action
|
120
|
+
end
|
121
|
+
|
122
|
+
def test_metric_data
|
123
|
+
response = invoke('metric_data?run_id=1',
|
124
|
+
{'Foo/Bar' => [1,2,3], 'Baz/Cux' => [4,5,6]})
|
125
|
+
|
126
|
+
assert_equal 1, response['Some/Metric/Spec']
|
127
|
+
post = @collector.agent_data[0]
|
128
|
+
assert_equal 'metric_data', post.action
|
129
|
+
assert_equal({'Foo/Bar' => [1,2,3], 'Baz/Cux' => [4,5,6]}, post.body)
|
130
|
+
assert_equal 1, post.run_id.to_i
|
131
|
+
end
|
132
|
+
|
133
|
+
def test_sql_trace_data
|
134
|
+
response = invoke('sql_trace_data?run_id=2',
|
135
|
+
['trace', 'trace', 'trace'])
|
136
|
+
|
137
|
+
assert_nil response
|
138
|
+
post = @collector.agent_data[0]
|
139
|
+
assert_equal 'sql_trace_data', post.action
|
140
|
+
assert_equal ['trace', 'trace', 'trace'], post.body
|
141
|
+
assert_equal 2, post.run_id.to_i
|
142
|
+
end
|
143
|
+
|
144
|
+
def test_transaction_sample_data
|
145
|
+
response = invoke('transaction_sample_data?run_id=3',
|
146
|
+
['node', ['node', 'node'], 'node'])
|
147
|
+
|
148
|
+
assert_nil response
|
149
|
+
post = @collector.agent_data[0]
|
150
|
+
assert_equal 'transaction_sample_data', post.action
|
151
|
+
assert_equal ['node', ['node', 'node'], 'node'], post.body
|
152
|
+
assert_equal 3, post.run_id.to_i
|
153
|
+
end
|
154
|
+
|
155
|
+
def test_error_data
|
156
|
+
response = invoke('error_data?run_id=4', ['error'])
|
157
|
+
|
158
|
+
assert_nil response
|
159
|
+
post = @collector.agent_data[0]
|
160
|
+
assert_equal 'error_data', post.action
|
161
|
+
assert_equal ['error'], post.body
|
162
|
+
end
|
163
|
+
|
164
|
+
def test_shutdown
|
165
|
+
response = invoke('shutdown?run_id=1')
|
166
|
+
|
167
|
+
assert_nil response
|
168
|
+
assert_equal 'shutdown', @collector.agent_data[0].action
|
169
|
+
end
|
170
|
+
|
171
|
+
def test_multiple_invokations
|
172
|
+
pid = Process.fork do
|
173
|
+
invoke('get_redirect_host')
|
174
|
+
invoke('connect')
|
175
|
+
invoke('metric_data?run_id=1')
|
176
|
+
invoke('transaction_sample_data?run_id=1')
|
177
|
+
invoke('shutdown?run_id=1')
|
178
|
+
end
|
179
|
+
invoke('get_redirect_host')
|
180
|
+
invoke('connect')
|
181
|
+
invoke('metric_data?run_id=2')
|
182
|
+
invoke('transaction_sample_data?run_id=2')
|
183
|
+
invoke('shutdown?run_id=2')
|
184
|
+
|
185
|
+
Process.wait(pid)
|
186
|
+
|
187
|
+
expected = ['get_redirect_host', 'connect', 'metric_data',
|
188
|
+
'transaction_sample_data', 'shutdown',
|
189
|
+
'get_redirect_host', 'connect', 'metric_data',
|
190
|
+
'transaction_sample_data', 'shutdown']
|
191
|
+
assert_equal expected.sort, @collector.agent_data.map(&:action).sort
|
192
|
+
end
|
193
|
+
|
194
|
+
def test_reset
|
195
|
+
@collector.mock['get_redirect_host'] = 'never!'
|
196
|
+
@collector.reset
|
197
|
+
assert_equal 'localhost', @collector.mock['get_redirect_host']
|
198
|
+
end
|
199
|
+
|
200
|
+
def invoke(method, post={}, code=200)
|
201
|
+
uri = URI.parse("http://127.0.0.1:30303/agent_listener/8/12345/#{method}")
|
202
|
+
request = Net::HTTP::Post.new("#{uri.path}?#{uri.query}")
|
203
|
+
request.body = Marshal.dump(post)
|
204
|
+
response = Net::HTTP.start(uri.host, uri.port) do |http|
|
205
|
+
http.request(request)
|
206
|
+
end
|
207
|
+
Marshal.load(response.body)
|
208
|
+
end
|
209
|
+
end
|
210
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
require 'ostruct'
|
2
|
+
|
3
|
+
module NewRelic
|
4
|
+
class FakeService
|
5
|
+
attr_accessor :request_timeout, :agent_id, :agent_data, :collector, :mock
|
6
|
+
|
7
|
+
def initialize
|
8
|
+
@agent_data = []
|
9
|
+
@supported_methods = [ :connect, :metric_data, :transaction_sample_data,
|
10
|
+
:error_data, :sql_trace_data, :shutdown ]
|
11
|
+
@collector = NewRelic::Control::Server.new(:name => 'fakehost', :port => 0)
|
12
|
+
@id_counter = 0
|
13
|
+
@base_expectations = {
|
14
|
+
'get_redirect_host' => 'localhost',
|
15
|
+
'connect' => { 'agent_run_id' => agent_run_id },
|
16
|
+
'metric_data' => { 'Some/Metric/Spec' => 1 },
|
17
|
+
'sql_trace_data' => nil,
|
18
|
+
'transaction_sample_data' => nil,
|
19
|
+
'error_data' => nil,
|
20
|
+
'shutdown' => nil,
|
21
|
+
}
|
22
|
+
reset
|
23
|
+
end
|
24
|
+
|
25
|
+
def agent_run_id
|
26
|
+
@id_counter += 1
|
27
|
+
end
|
28
|
+
|
29
|
+
def reset
|
30
|
+
@mock = @base_expectations.dup
|
31
|
+
@id_counter = 0
|
32
|
+
@agent_data = []
|
33
|
+
end
|
34
|
+
|
35
|
+
def method_missing(method, *args)
|
36
|
+
if @supported_methods.include?(method)
|
37
|
+
@agent_data << OpenStruct.new(:method => method, :params => args)
|
38
|
+
@mock[method.to_s]
|
39
|
+
else
|
40
|
+
super
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|