appsignal 2.11.1-java → 3.0.0.beta.1-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +6 -1
- data/.semaphore/semaphore.yml +197 -23
- data/CHANGELOG.md +25 -0
- data/README.md +9 -0
- data/Rakefile +9 -6
- data/build_matrix.yml +13 -4
- data/ext/agent.yml +17 -17
- data/ext/base.rb +12 -9
- data/gemfiles/no_dependencies.gemfile +7 -0
- data/gemfiles/resque-2.gemfile +0 -1
- data/gemfiles/webmachine.gemfile +1 -0
- data/lib/appsignal.rb +1 -27
- data/lib/appsignal/auth_check.rb +2 -8
- data/lib/appsignal/cli.rb +1 -23
- data/lib/appsignal/cli/diagnose/utils.rb +8 -11
- data/lib/appsignal/cli/install.rb +5 -8
- data/lib/appsignal/config.rb +0 -24
- data/lib/appsignal/event_formatter.rb +0 -25
- data/lib/appsignal/helpers/instrumentation.rb +32 -0
- data/lib/appsignal/hooks.rb +0 -23
- data/lib/appsignal/hooks/action_cable.rb +3 -34
- data/lib/appsignal/hooks/active_support_notifications.rb +7 -86
- data/lib/appsignal/hooks/celluloid.rb +5 -9
- data/lib/appsignal/hooks/net_http.rb +2 -12
- data/lib/appsignal/hooks/puma.rb +3 -5
- data/lib/appsignal/hooks/que.rb +1 -1
- data/lib/appsignal/hooks/rake.rb +2 -24
- data/lib/appsignal/hooks/redis.rb +2 -13
- data/lib/appsignal/hooks/resque.rb +2 -43
- data/lib/appsignal/hooks/shoryuken.rb +43 -4
- data/lib/appsignal/hooks/unicorn.rb +3 -24
- data/lib/appsignal/hooks/webmachine.rb +1 -7
- data/lib/appsignal/integrations/action_cable.rb +34 -0
- data/lib/appsignal/integrations/active_support_notifications.rb +77 -0
- data/lib/appsignal/integrations/net_http.rb +16 -0
- data/lib/appsignal/integrations/object.rb +44 -17
- data/lib/appsignal/integrations/padrino.rb +5 -7
- data/lib/appsignal/integrations/que.rb +26 -33
- data/lib/appsignal/integrations/railtie.rb +1 -4
- data/lib/appsignal/integrations/rake.rb +26 -2
- data/lib/appsignal/integrations/redis.rb +17 -0
- data/lib/appsignal/integrations/resque.rb +39 -10
- data/lib/appsignal/integrations/unicorn.rb +28 -0
- data/lib/appsignal/integrations/webmachine.rb +22 -24
- data/lib/appsignal/minutely.rb +0 -12
- data/lib/appsignal/system.rb +4 -0
- data/lib/appsignal/transaction.rb +30 -2
- data/lib/appsignal/version.rb +1 -1
- data/spec/lib/appsignal/auth_check_spec.rb +1 -24
- data/spec/lib/appsignal/cli_spec.rb +1 -1
- data/spec/lib/appsignal/config_spec.rb +0 -66
- data/spec/lib/appsignal/event_formatter_spec.rb +0 -37
- data/spec/lib/appsignal/hooks/celluloid_spec.rb +6 -1
- data/spec/lib/appsignal/hooks/rake_spec.rb +1 -2
- data/spec/lib/appsignal/hooks/redis_spec.rb +50 -15
- data/spec/lib/appsignal/hooks/resque_spec.rb +10 -2
- data/spec/lib/appsignal/hooks/shoryuken_spec.rb +151 -104
- data/spec/lib/appsignal/hooks/sidekiq_spec.rb +4 -2
- data/spec/lib/appsignal/hooks/unicorn_spec.rb +14 -3
- data/spec/lib/appsignal/hooks/webmachine_spec.rb +2 -13
- data/spec/lib/appsignal/hooks_spec.rb +0 -57
- data/spec/lib/appsignal/integrations/object_spec.rb +25 -10
- data/spec/lib/appsignal/integrations/padrino_spec.rb +2 -3
- data/spec/lib/appsignal/integrations/railtie_spec.rb +0 -45
- data/spec/lib/appsignal/integrations/webmachine_spec.rb +26 -8
- data/spec/lib/appsignal/minutely_spec.rb +0 -19
- data/spec/lib/appsignal/transaction_spec.rb +56 -14
- data/spec/lib/appsignal/transmitter_spec.rb +1 -1
- data/spec/lib/appsignal_spec.rb +30 -69
- data/spec/spec_helper.rb +1 -15
- metadata +13 -22
- data/lib/appsignal/cli/notify_of_deploy.rb +0 -131
- data/lib/appsignal/integrations/resque_active_job.rb +0 -19
- data/lib/appsignal/js_exception_transaction.rb +0 -56
- data/lib/appsignal/rack/js_exception_catcher.rb +0 -80
- data/spec/lib/appsignal/cli/notify_of_deploy_spec.rb +0 -180
- data/spec/lib/appsignal/integrations/resque_active_job_spec.rb +0 -28
- data/spec/lib/appsignal/integrations/resque_spec.rb +0 -28
- data/spec/lib/appsignal/js_exception_transaction_spec.rb +0 -128
- data/spec/lib/appsignal/rack/js_exception_catcher_spec.rb +0 -170
@@ -6,8 +6,6 @@ module Appsignal
|
|
6
6
|
class ActiveSupportNotificationsHook < Appsignal::Hooks::Hook
|
7
7
|
register :active_support_notifications
|
8
8
|
|
9
|
-
BANG = "!".freeze
|
10
|
-
|
11
9
|
def dependencies_present?
|
12
10
|
defined?(::ActiveSupport::Notifications::Instrumenter)
|
13
11
|
end
|
@@ -23,100 +21,23 @@ module Appsignal
|
|
23
21
|
end
|
24
22
|
end
|
25
23
|
|
24
|
+
require "appsignal/integrations/active_support_notifications"
|
26
25
|
instrumenter = ::ActiveSupport::Notifications::Instrumenter
|
27
|
-
|
26
|
+
parent_integration_module = Appsignal::Integrations::ActiveSupportNotificationsIntegration
|
28
27
|
if instrumenter.method_defined?(:start) && instrumenter.method_defined?(:finish)
|
29
|
-
|
28
|
+
install_module(parent_integration_module::StartFinishIntegration)
|
30
29
|
else
|
31
|
-
|
30
|
+
install_module(parent_integration_module::InstrumentIntegration)
|
32
31
|
end
|
33
32
|
|
34
33
|
# rubocop:disable Style/GuardClause
|
35
34
|
if instrumenter.method_defined?(:finish_with_state)
|
36
|
-
|
35
|
+
install_module(parent_integration_module::FinishStateIntegration)
|
37
36
|
end
|
38
37
|
end
|
39
38
|
|
40
|
-
def
|
41
|
-
::ActiveSupport::Notifications::Instrumenter.
|
42
|
-
alias instrument_without_appsignal instrument
|
43
|
-
|
44
|
-
def instrument(name, payload = {}, &block)
|
45
|
-
# Events that start with a bang are internal to Rails
|
46
|
-
instrument_this = name[0] != BANG
|
47
|
-
|
48
|
-
Appsignal::Transaction.current.start_event if instrument_this
|
49
|
-
|
50
|
-
instrument_without_appsignal(name, payload, &block)
|
51
|
-
ensure
|
52
|
-
if instrument_this
|
53
|
-
title, body, body_format = Appsignal::EventFormatter.format(name, payload)
|
54
|
-
Appsignal::Transaction.current.finish_event(
|
55
|
-
name.to_s,
|
56
|
-
title,
|
57
|
-
body,
|
58
|
-
body_format
|
59
|
-
)
|
60
|
-
end
|
61
|
-
end
|
62
|
-
end
|
63
|
-
end
|
64
|
-
|
65
|
-
def install_start_finish
|
66
|
-
::ActiveSupport::Notifications::Instrumenter.class_eval do
|
67
|
-
alias start_without_appsignal start
|
68
|
-
|
69
|
-
def start(name, payload = {})
|
70
|
-
# Events that start with a bang are internal to Rails
|
71
|
-
instrument_this = name[0] != BANG
|
72
|
-
|
73
|
-
Appsignal::Transaction.current.start_event if instrument_this
|
74
|
-
|
75
|
-
start_without_appsignal(name, payload)
|
76
|
-
end
|
77
|
-
|
78
|
-
alias finish_without_appsignal finish
|
79
|
-
|
80
|
-
def finish(name, payload = {})
|
81
|
-
# Events that start with a bang are internal to Rails
|
82
|
-
instrument_this = name[0] != BANG
|
83
|
-
|
84
|
-
if instrument_this
|
85
|
-
title, body, body_format = Appsignal::EventFormatter.format(name, payload)
|
86
|
-
Appsignal::Transaction.current.finish_event(
|
87
|
-
name.to_s,
|
88
|
-
title,
|
89
|
-
body,
|
90
|
-
body_format
|
91
|
-
)
|
92
|
-
end
|
93
|
-
|
94
|
-
finish_without_appsignal(name, payload)
|
95
|
-
end
|
96
|
-
end
|
97
|
-
end
|
98
|
-
|
99
|
-
def install_finish_with_state
|
100
|
-
::ActiveSupport::Notifications::Instrumenter.class_eval do
|
101
|
-
alias finish_with_state_without_appsignal finish_with_state
|
102
|
-
|
103
|
-
def finish_with_state(listeners_state, name, payload = {})
|
104
|
-
# Events that start with a bang are internal to Rails
|
105
|
-
instrument_this = name[0] != BANG
|
106
|
-
|
107
|
-
if instrument_this
|
108
|
-
title, body, body_format = Appsignal::EventFormatter.format(name, payload)
|
109
|
-
Appsignal::Transaction.current.finish_event(
|
110
|
-
name.to_s,
|
111
|
-
title,
|
112
|
-
body,
|
113
|
-
body_format
|
114
|
-
)
|
115
|
-
end
|
116
|
-
|
117
|
-
finish_with_state_without_appsignal(listeners_state, name, payload)
|
118
|
-
end
|
119
|
-
end
|
39
|
+
def install_module(mod)
|
40
|
+
::ActiveSupport::Notifications::Instrumenter.send(:prepend, mod)
|
120
41
|
end
|
121
42
|
end
|
122
43
|
end
|
@@ -16,16 +16,12 @@ module Appsignal
|
|
16
16
|
# down Celluloid so we're sure our thread does not aggravate this situation.
|
17
17
|
# This way we also make sure any outstanding transactions get flushed.
|
18
18
|
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
def shutdown
|
24
|
-
Appsignal.stop("celluloid")
|
25
|
-
shutdown_without_appsignal
|
26
|
-
end
|
19
|
+
Celluloid.singleton_class.send(:prepend, Module.new do
|
20
|
+
def shutdown
|
21
|
+
Appsignal.stop("celluloid")
|
22
|
+
super
|
27
23
|
end
|
28
|
-
end
|
24
|
+
end)
|
29
25
|
end
|
30
26
|
end
|
31
27
|
end
|
@@ -13,18 +13,8 @@ module Appsignal
|
|
13
13
|
end
|
14
14
|
|
15
15
|
def install
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
def request(request, body = nil, &block)
|
20
|
-
Appsignal.instrument(
|
21
|
-
"request.net_http",
|
22
|
-
"#{request.method} #{use_ssl? ? "https" : "http"}://#{request["host"] || address}"
|
23
|
-
) do
|
24
|
-
request_without_appsignal(request, body, &block)
|
25
|
-
end
|
26
|
-
end
|
27
|
-
end
|
16
|
+
require "appsignal/integrations/net_http"
|
17
|
+
Net::HTTP.send(:prepend, Appsignal::Integrations::NetHttpIntegration)
|
28
18
|
|
29
19
|
Appsignal::Environment.report_enabled("net_http")
|
30
20
|
end
|
data/lib/appsignal/hooks/puma.rb
CHANGED
@@ -29,14 +29,12 @@ module Appsignal
|
|
29
29
|
|
30
30
|
return unless defined?(::Puma::Cluster)
|
31
31
|
# For clustered mode with multiple workers
|
32
|
-
::Puma::Cluster.
|
33
|
-
alias stop_workers_without_appsignal stop_workers
|
34
|
-
|
32
|
+
::Puma::Cluster.send(:prepend, Module.new do
|
35
33
|
def stop_workers
|
36
34
|
Appsignal.stop("puma cluster")
|
37
|
-
|
35
|
+
super
|
38
36
|
end
|
39
|
-
end
|
37
|
+
end)
|
40
38
|
end
|
41
39
|
end
|
42
40
|
end
|
data/lib/appsignal/hooks/que.rb
CHANGED
@@ -12,7 +12,7 @@ module Appsignal
|
|
12
12
|
|
13
13
|
def install
|
14
14
|
require "appsignal/integrations/que"
|
15
|
-
::Que::Job.send(:
|
15
|
+
::Que::Job.send(:prepend, Appsignal::Integrations::QuePlugin)
|
16
16
|
|
17
17
|
::Que.error_notifier = proc do |error, _job|
|
18
18
|
Appsignal::Transaction.current.set_error(error)
|
data/lib/appsignal/hooks/rake.rb
CHANGED
@@ -11,30 +11,8 @@ module Appsignal
|
|
11
11
|
end
|
12
12
|
|
13
13
|
def install
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
def execute(*args)
|
18
|
-
execute_without_appsignal(*args)
|
19
|
-
rescue Exception => error # rubocop:disable Lint/RescueException
|
20
|
-
# Format given arguments and cast to hash if possible
|
21
|
-
params, _ = args
|
22
|
-
params = params.to_hash if params.respond_to?(:to_hash)
|
23
|
-
|
24
|
-
transaction = Appsignal::Transaction.create(
|
25
|
-
SecureRandom.uuid,
|
26
|
-
Appsignal::Transaction::BACKGROUND_JOB,
|
27
|
-
Appsignal::Transaction::GenericRequest.new(
|
28
|
-
:params => params
|
29
|
-
)
|
30
|
-
)
|
31
|
-
transaction.set_action(name)
|
32
|
-
transaction.set_error(error)
|
33
|
-
transaction.complete
|
34
|
-
Appsignal.stop("rake")
|
35
|
-
raise error
|
36
|
-
end
|
37
|
-
end
|
14
|
+
require "appsignal/integrations/rake"
|
15
|
+
::Rake::Task.send(:prepend, Appsignal::Integrations::RakeIntegration)
|
38
16
|
end
|
39
17
|
end
|
40
18
|
end
|
@@ -13,19 +13,8 @@ module Appsignal
|
|
13
13
|
end
|
14
14
|
|
15
15
|
def install
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
def process(commands, &block)
|
20
|
-
sanitized_commands = commands.map do |command, *args|
|
21
|
-
"#{command}#{" ?" * args.size}"
|
22
|
-
end.join("\n")
|
23
|
-
|
24
|
-
Appsignal.instrument "query.redis", id, sanitized_commands do
|
25
|
-
process_without_appsignal(commands, &block)
|
26
|
-
end
|
27
|
-
end
|
28
|
-
end
|
16
|
+
require "appsignal/integrations/redis"
|
17
|
+
::Redis::Client.send(:prepend, Appsignal::Integrations::RedisIntegration)
|
29
18
|
|
30
19
|
Appsignal::Environment.report_enabled("redis")
|
31
20
|
end
|
@@ -11,49 +11,8 @@ module Appsignal
|
|
11
11
|
end
|
12
12
|
|
13
13
|
def install
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
def perform
|
18
|
-
transaction = Appsignal::Transaction.create(
|
19
|
-
SecureRandom.uuid,
|
20
|
-
Appsignal::Transaction::BACKGROUND_JOB,
|
21
|
-
Appsignal::Transaction::GenericRequest.new({})
|
22
|
-
)
|
23
|
-
|
24
|
-
Appsignal.instrument "perform.resque" do
|
25
|
-
perform_without_appsignal
|
26
|
-
end
|
27
|
-
rescue Exception => exception # rubocop:disable Lint/RescueException
|
28
|
-
transaction.set_error(exception)
|
29
|
-
raise exception
|
30
|
-
ensure
|
31
|
-
if transaction
|
32
|
-
transaction.set_action_if_nil("#{payload["class"]}#perform")
|
33
|
-
args =
|
34
|
-
Appsignal::Utils::HashSanitizer.sanitize(
|
35
|
-
ResqueHelpers.arguments(payload),
|
36
|
-
Appsignal.config[:filter_parameters]
|
37
|
-
)
|
38
|
-
transaction.params = args if args
|
39
|
-
transaction.set_tags("queue" => queue)
|
40
|
-
|
41
|
-
Appsignal::Transaction.complete_current!
|
42
|
-
end
|
43
|
-
Appsignal.stop("resque")
|
44
|
-
end
|
45
|
-
end
|
46
|
-
end
|
47
|
-
|
48
|
-
class ResqueHelpers
|
49
|
-
def self.arguments(payload)
|
50
|
-
case payload["class"]
|
51
|
-
when "ActiveJob::QueueAdapters::ResqueAdapter::JobWrapper"
|
52
|
-
nil # Set in the ActiveJob integration
|
53
|
-
else
|
54
|
-
payload["args"]
|
55
|
-
end
|
56
|
-
end
|
14
|
+
require "appsignal/integrations/resque"
|
15
|
+
Resque::Job.send(:prepend, Appsignal::Integrations::ResqueIntegration)
|
57
16
|
end
|
58
17
|
end
|
59
18
|
end
|
@@ -5,21 +5,60 @@ module Appsignal
|
|
5
5
|
# @api private
|
6
6
|
class ShoryukenMiddleware
|
7
7
|
def call(worker_instance, queue, sqs_msg, body)
|
8
|
-
|
8
|
+
batch = sqs_msg.is_a?(Array)
|
9
|
+
attributes =
|
10
|
+
if batch
|
11
|
+
# We can't instrument batched message separately, the `yield` will
|
12
|
+
# perform all the batched messages.
|
13
|
+
# To provide somewhat useful metadata, Get first message based on
|
14
|
+
# SentTimestamp, and use its attributes as metadata for the
|
15
|
+
# transaction. We can't combine them all because then they would
|
16
|
+
# overwrite each other and the last message (in an sorted order)
|
17
|
+
# would be used as the source of the metadata. With the
|
18
|
+
# oldest/first message at least some useful information is stored
|
19
|
+
# such as the first received time and the number of retries for the
|
20
|
+
# first message. The newer message should have lower values and
|
21
|
+
# timestamps in their metadata.
|
22
|
+
first_msg = sqs_msg.min do |a, b|
|
23
|
+
a.attributes["SentTimestamp"].to_i <=> b.attributes["SentTimestamp"].to_i
|
24
|
+
end
|
25
|
+
# Add batch => true metadata so people can recognize when a
|
26
|
+
# transaction is about a batch of messages.
|
27
|
+
first_msg.attributes.merge(:batch => true)
|
28
|
+
else
|
29
|
+
sqs_msg.attributes.merge(:message_id => sqs_msg.message_id)
|
30
|
+
end
|
31
|
+
metadata = { :queue => queue }.merge(attributes)
|
9
32
|
options = {
|
10
33
|
:class => worker_instance.class.name,
|
11
34
|
:method => "perform",
|
12
35
|
:metadata => metadata
|
13
36
|
}
|
14
37
|
|
15
|
-
args =
|
38
|
+
args =
|
39
|
+
if batch
|
40
|
+
bodies = {}
|
41
|
+
sqs_msg.each_with_index do |msg, index|
|
42
|
+
# Store all separate bodies on a hash with the key being the
|
43
|
+
# message_id
|
44
|
+
bodies[msg.message_id] = body[index]
|
45
|
+
end
|
46
|
+
bodies
|
47
|
+
else
|
48
|
+
case body
|
49
|
+
when Hash
|
50
|
+
body
|
51
|
+
else
|
52
|
+
{ :params => body }
|
53
|
+
end
|
54
|
+
end
|
16
55
|
options[:params] = Appsignal::Utils::HashSanitizer.sanitize(
|
17
56
|
args,
|
18
57
|
Appsignal.config[:filter_parameters]
|
19
58
|
)
|
20
59
|
|
21
|
-
if
|
22
|
-
options[:queue_start] = Time.at(
|
60
|
+
if attributes.key?("SentTimestamp")
|
61
|
+
options[:queue_start] = Time.at(attributes["SentTimestamp"].to_i / 1000)
|
23
62
|
end
|
24
63
|
|
25
64
|
Appsignal.monitor_transaction("perform_job.shoryuken", options) do
|
@@ -12,30 +12,9 @@ module Appsignal
|
|
12
12
|
end
|
13
13
|
|
14
14
|
def install
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
# We'd love to be able to hook this into Unicorn in a less
|
19
|
-
# intrusive way, but this is the best we can do given the
|
20
|
-
# options we have.
|
21
|
-
|
22
|
-
::Unicorn::HttpServer.class_eval do
|
23
|
-
alias worker_loop_without_appsignal worker_loop
|
24
|
-
|
25
|
-
def worker_loop(worker)
|
26
|
-
Appsignal.forked
|
27
|
-
worker_loop_without_appsignal(worker)
|
28
|
-
end
|
29
|
-
end
|
30
|
-
|
31
|
-
::Unicorn::Worker.class_eval do
|
32
|
-
alias close_without_appsignal close
|
33
|
-
|
34
|
-
def close
|
35
|
-
Appsignal.stop("unicorn")
|
36
|
-
close_without_appsignal
|
37
|
-
end
|
38
|
-
end
|
15
|
+
require "appsignal/integrations/unicorn"
|
16
|
+
::Unicorn::HttpServer.send(:prepend, Appsignal::Integrations::UnicornIntegration::Server)
|
17
|
+
::Unicorn::Worker.send(:prepend, Appsignal::Integrations::UnicornIntegration::Worker)
|
39
18
|
end
|
40
19
|
end
|
41
20
|
end
|
@@ -12,13 +12,7 @@ module Appsignal
|
|
12
12
|
|
13
13
|
def install
|
14
14
|
require "appsignal/integrations/webmachine"
|
15
|
-
::Webmachine::Decision::FSM.
|
16
|
-
include Appsignal::Integrations::WebmachinePlugin::FSM
|
17
|
-
alias run_without_appsignal run
|
18
|
-
alias run run_with_appsignal
|
19
|
-
alias handle_exceptions_without_appsignal handle_exceptions
|
20
|
-
alias handle_exceptions handle_exceptions_with_appsignal
|
21
|
-
end
|
15
|
+
::Webmachine::Decision::FSM.send(:prepend, Appsignal::Integrations::WebmachineIntegration)
|
22
16
|
end
|
23
17
|
end
|
24
18
|
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Appsignal
|
4
|
+
module Integrations
|
5
|
+
module ActionCableIntegration
|
6
|
+
def perform_action(*args, &block)
|
7
|
+
# The request is only the original websocket request
|
8
|
+
env = connection.env
|
9
|
+
request = ActionDispatch::Request.new(env)
|
10
|
+
env[Appsignal::Hooks::ActionCableHook::REQUEST_ID] ||=
|
11
|
+
request.request_id || SecureRandom.uuid
|
12
|
+
|
13
|
+
transaction = Appsignal::Transaction.create(
|
14
|
+
env[Appsignal::Hooks::ActionCableHook::REQUEST_ID],
|
15
|
+
Appsignal::Transaction::ACTION_CABLE,
|
16
|
+
request
|
17
|
+
)
|
18
|
+
|
19
|
+
begin
|
20
|
+
super
|
21
|
+
rescue Exception => exception # rubocop:disable Lint/RescueException
|
22
|
+
transaction.set_error(exception)
|
23
|
+
raise exception
|
24
|
+
ensure
|
25
|
+
transaction.params = args.first
|
26
|
+
transaction.set_action_if_nil("#{self.class}##{args.first["action"]}")
|
27
|
+
transaction.set_metadata("path", request.path)
|
28
|
+
transaction.set_metadata("method", "websocket")
|
29
|
+
Appsignal::Transaction.complete_current!
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|