appsignal 2.11.0.alpha.2-java → 2.11.0.beta.5-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +3 -0
- data/.semaphore/semaphore.yml +94 -10
- data/CHANGELOG.md +31 -1
- data/README.md +4 -4
- data/Rakefile +16 -4
- data/appsignal.gemspec +1 -1
- data/build_matrix.yml +7 -3
- data/ext/Rakefile +2 -0
- data/ext/agent.yml +19 -19
- data/ext/base.rb +7 -0
- data/ext/extconf.rb +2 -0
- data/gemfiles/rails-4.2.gemfile +9 -2
- data/gemfiles/rails-5.0.gemfile +1 -0
- data/gemfiles/rails-5.1.gemfile +1 -0
- data/gemfiles/rails-5.2.gemfile +1 -0
- data/gemfiles/rails-6.0.gemfile +1 -0
- data/gemfiles/resque-1.gemfile +7 -0
- data/gemfiles/{resque.gemfile → resque-2.gemfile} +1 -1
- data/lib/appsignal.rb +1 -0
- data/lib/appsignal/auth_check.rb +4 -2
- data/lib/appsignal/cli/diagnose.rb +1 -1
- data/lib/appsignal/config.rb +35 -2
- data/lib/appsignal/extension.rb +6 -5
- data/lib/appsignal/extension/jruby.rb +6 -5
- data/lib/appsignal/hooks.rb +25 -0
- data/lib/appsignal/hooks/active_job.rb +137 -0
- data/lib/appsignal/hooks/puma.rb +0 -1
- data/lib/appsignal/hooks/resque.rb +60 -0
- data/lib/appsignal/hooks/sidekiq.rb +17 -94
- data/lib/appsignal/integrations/delayed_job_plugin.rb +1 -1
- data/lib/appsignal/integrations/que.rb +1 -1
- data/lib/appsignal/integrations/resque.rb +9 -12
- data/lib/appsignal/integrations/resque_active_job.rb +9 -32
- data/lib/appsignal/probes.rb +7 -0
- data/lib/appsignal/probes/puma.rb +1 -1
- data/lib/appsignal/probes/sidekiq.rb +3 -1
- data/lib/appsignal/transaction.rb +10 -0
- data/lib/appsignal/utils/deprecation_message.rb +6 -2
- data/lib/appsignal/version.rb +1 -1
- data/spec/lib/appsignal/auth_check_spec.rb +23 -0
- data/spec/lib/appsignal/capistrano2_spec.rb +1 -1
- data/spec/lib/appsignal/capistrano3_spec.rb +1 -1
- data/spec/lib/appsignal/cli/diagnose_spec.rb +42 -0
- data/spec/lib/appsignal/config_spec.rb +21 -0
- data/spec/lib/appsignal/extension/jruby_spec.rb +31 -28
- data/spec/lib/appsignal/extension_install_failure_spec.rb +23 -0
- data/spec/lib/appsignal/hooks/activejob_spec.rb +591 -0
- data/spec/lib/appsignal/hooks/delayed_job_spec.rb +3 -14
- data/spec/lib/appsignal/hooks/resque_spec.rb +185 -0
- data/spec/lib/appsignal/hooks/sidekiq_spec.rb +222 -268
- data/spec/lib/appsignal/hooks_spec.rb +57 -0
- data/spec/lib/appsignal/integrations/que_spec.rb +25 -6
- data/spec/lib/appsignal/integrations/resque_active_job_spec.rb +20 -179
- data/spec/lib/appsignal/integrations/resque_spec.rb +20 -85
- data/spec/lib/appsignal/marker_spec.rb +1 -1
- data/spec/lib/appsignal/probes/sidekiq_spec.rb +10 -7
- data/spec/lib/appsignal/transaction_spec.rb +5 -7
- data/spec/spec_helper.rb +5 -0
- data/spec/support/helpers/action_mailer_helpers.rb +25 -0
- data/spec/support/helpers/config_helpers.rb +3 -2
- data/spec/support/helpers/dependency_helper.rb +9 -2
- data/spec/support/helpers/transaction_helpers.rb +6 -0
- data/spec/support/stubs/sidekiq/api.rb +1 -1
- data/spec/support/testing.rb +19 -19
- metadata +16 -4
data/lib/appsignal/hooks.rb
CHANGED
@@ -69,10 +69,34 @@ module Appsignal
|
|
69
69
|
text.size > 200 ? "#{text[0...197]}..." : text
|
70
70
|
end
|
71
71
|
end
|
72
|
+
|
73
|
+
# Alias Probes constants that have moved to their own module in version
|
74
|
+
# 2.11.0.
|
75
|
+
def self.const_missing(name)
|
76
|
+
case name
|
77
|
+
when :SidekiqProbe
|
78
|
+
callers = caller
|
79
|
+
Appsignal::Utils::DeprecationMessage.message \
|
80
|
+
"The constant Appsignal::Hooks::SidekiqProbe has been deprecated. " \
|
81
|
+
"Please update the constant name to Appsignal::Probes::SidekiqProbe " \
|
82
|
+
"in the following file to remove this message.\n#{callers.first}"
|
83
|
+
Appsignal::Probes::SidekiqProbe
|
84
|
+
when :PumaProbe
|
85
|
+
callers = caller
|
86
|
+
Appsignal::Utils::DeprecationMessage.message \
|
87
|
+
"The constant Appsignal::Hooks::PumaProbe has been deprecated. " \
|
88
|
+
"Please update the constant name to Appsignal::Probes::PumaProbe " \
|
89
|
+
"in the following file to remove this message.\n#{callers.first}"
|
90
|
+
Appsignal::Probes::PumaProbe
|
91
|
+
else
|
92
|
+
super
|
93
|
+
end
|
94
|
+
end
|
72
95
|
end
|
73
96
|
end
|
74
97
|
|
75
98
|
require "appsignal/hooks/action_cable"
|
99
|
+
require "appsignal/hooks/active_job"
|
76
100
|
require "appsignal/hooks/active_support_notifications"
|
77
101
|
require "appsignal/hooks/celluloid"
|
78
102
|
require "appsignal/hooks/delayed_job"
|
@@ -81,6 +105,7 @@ require "appsignal/hooks/passenger"
|
|
81
105
|
require "appsignal/hooks/puma"
|
82
106
|
require "appsignal/hooks/rake"
|
83
107
|
require "appsignal/hooks/redis"
|
108
|
+
require "appsignal/hooks/resque"
|
84
109
|
require "appsignal/hooks/sequel"
|
85
110
|
require "appsignal/hooks/shoryuken"
|
86
111
|
require "appsignal/hooks/sidekiq"
|
@@ -0,0 +1,137 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Appsignal
|
4
|
+
class Hooks
|
5
|
+
# @api private
|
6
|
+
class ActiveJobHook < Appsignal::Hooks::Hook
|
7
|
+
register :active_job
|
8
|
+
|
9
|
+
def dependencies_present?
|
10
|
+
defined?(::ActiveJob)
|
11
|
+
end
|
12
|
+
|
13
|
+
def install
|
14
|
+
::ActiveJob::Base
|
15
|
+
.extend ::Appsignal::Hooks::ActiveJobHook::ActiveJobClassInstrumentation
|
16
|
+
end
|
17
|
+
|
18
|
+
module ActiveJobClassInstrumentation
|
19
|
+
def execute(job)
|
20
|
+
job_status = nil
|
21
|
+
current_transaction = Appsignal::Transaction.current
|
22
|
+
transaction =
|
23
|
+
if current_transaction.nil_transaction?
|
24
|
+
# No standalone integration started before ActiveJob integration.
|
25
|
+
# We don't have a separate integration for this QueueAdapter like
|
26
|
+
# we do for Sidekiq.
|
27
|
+
#
|
28
|
+
# Prefer job_id from provider, instead of ActiveJob's internal ID.
|
29
|
+
Appsignal::Transaction.create(
|
30
|
+
job["provider_job_id"] || job["job_id"],
|
31
|
+
Appsignal::Transaction::BACKGROUND_JOB,
|
32
|
+
Appsignal::Transaction::GenericRequest.new({})
|
33
|
+
)
|
34
|
+
else
|
35
|
+
current_transaction
|
36
|
+
end
|
37
|
+
|
38
|
+
super
|
39
|
+
rescue Exception => exception # rubocop:disable Lint/RescueException
|
40
|
+
job_status = :failed
|
41
|
+
transaction.set_error(exception)
|
42
|
+
raise exception
|
43
|
+
ensure
|
44
|
+
if transaction
|
45
|
+
transaction.params =
|
46
|
+
Appsignal::Utils::HashSanitizer.sanitize(
|
47
|
+
job["arguments"],
|
48
|
+
Appsignal.config[:filter_parameters]
|
49
|
+
)
|
50
|
+
|
51
|
+
transaction_tags = ActiveJobHelpers.transaction_tags_for(job)
|
52
|
+
transaction_tags["active_job_id"] = job["job_id"]
|
53
|
+
provider_job_id = job["provider_job_id"]
|
54
|
+
if provider_job_id
|
55
|
+
transaction_tags[:provider_job_id] = provider_job_id
|
56
|
+
end
|
57
|
+
transaction.set_tags(transaction_tags)
|
58
|
+
|
59
|
+
transaction.set_action_if_nil(ActiveJobHelpers.action_name(job))
|
60
|
+
enqueued_at = job["enqueued_at"]
|
61
|
+
if enqueued_at # Present in Rails 6 and up
|
62
|
+
transaction.set_queue_start((Time.parse(enqueued_at).to_f * 1_000).to_i)
|
63
|
+
end
|
64
|
+
|
65
|
+
if current_transaction.nil_transaction?
|
66
|
+
# Only complete transaction if ActiveJob is not wrapped in
|
67
|
+
# another supported integration, such as Sidekiq.
|
68
|
+
Appsignal::Transaction.complete_current!
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
metrics = ActiveJobHelpers.metrics_for(job)
|
73
|
+
metrics.each do |(metric_name, tags)|
|
74
|
+
if job_status
|
75
|
+
ActiveJobHelpers.increment_counter metric_name, 1,
|
76
|
+
tags.merge(:status => job_status)
|
77
|
+
end
|
78
|
+
ActiveJobHelpers.increment_counter metric_name, 1,
|
79
|
+
tags.merge(:status => :processed)
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
module ActiveJobHelpers
|
85
|
+
ACTION_MAILER_CLASSES = [
|
86
|
+
"ActionMailer::DeliveryJob",
|
87
|
+
"ActionMailer::Parameterized::DeliveryJob",
|
88
|
+
"ActionMailer::MailDeliveryJob"
|
89
|
+
].freeze
|
90
|
+
|
91
|
+
def self.action_name(job)
|
92
|
+
case job["job_class"]
|
93
|
+
when *ACTION_MAILER_CLASSES
|
94
|
+
job["arguments"][0..1].join("#")
|
95
|
+
else
|
96
|
+
"#{job["job_class"]}#perform"
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
# Returns an array of metrics with tags used to report the job metrics
|
101
|
+
#
|
102
|
+
# If job ONLY has a queue, it will return `queue_job_count` with tags.
|
103
|
+
# If job has a queue AND priority, it will ALSO return
|
104
|
+
# `queue_priority_job_count` with tags.
|
105
|
+
#
|
106
|
+
# @return [Array] Array of metrics with tags to report.
|
107
|
+
def self.metrics_for(job)
|
108
|
+
tags = { :queue => job["queue_name"] }
|
109
|
+
metrics = [["queue_job_count", tags]]
|
110
|
+
|
111
|
+
priority = job["priority"]
|
112
|
+
if priority
|
113
|
+
metrics << [
|
114
|
+
"queue_priority_job_count",
|
115
|
+
tags.merge(:priority => priority)
|
116
|
+
]
|
117
|
+
end
|
118
|
+
|
119
|
+
metrics
|
120
|
+
end
|
121
|
+
|
122
|
+
def self.transaction_tags_for(job)
|
123
|
+
tags = {}
|
124
|
+
queue = job["queue_name"]
|
125
|
+
tags[:queue] = queue if queue
|
126
|
+
priority = job["priority"]
|
127
|
+
tags[:priority] = priority if priority
|
128
|
+
tags
|
129
|
+
end
|
130
|
+
|
131
|
+
def self.increment_counter(key, value, tags = {})
|
132
|
+
Appsignal.increment_counter "active_job_#{key}", value, tags
|
133
|
+
end
|
134
|
+
end
|
135
|
+
end
|
136
|
+
end
|
137
|
+
end
|
data/lib/appsignal/hooks/puma.rb
CHANGED
@@ -0,0 +1,60 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Appsignal
|
4
|
+
class Hooks
|
5
|
+
# @api private
|
6
|
+
class ResqueHook < Appsignal::Hooks::Hook
|
7
|
+
register :resque
|
8
|
+
|
9
|
+
def dependencies_present?
|
10
|
+
defined?(::Resque)
|
11
|
+
end
|
12
|
+
|
13
|
+
def install
|
14
|
+
Resque::Job.class_eval do
|
15
|
+
alias_method :perform_without_appsignal, :perform
|
16
|
+
|
17
|
+
def perform
|
18
|
+
transaction = Appsignal::Transaction.create(
|
19
|
+
SecureRandom.uuid,
|
20
|
+
Appsignal::Transaction::BACKGROUND_JOB,
|
21
|
+
Appsignal::Transaction::GenericRequest.new({})
|
22
|
+
)
|
23
|
+
|
24
|
+
Appsignal.instrument "perform.resque" do
|
25
|
+
perform_without_appsignal
|
26
|
+
end
|
27
|
+
rescue Exception => exception # rubocop:disable Lint/RescueException
|
28
|
+
transaction.set_error(exception)
|
29
|
+
raise exception
|
30
|
+
ensure
|
31
|
+
if transaction
|
32
|
+
transaction.set_action_if_nil("#{payload["class"]}#perform")
|
33
|
+
args =
|
34
|
+
Appsignal::Utils::HashSanitizer.sanitize(
|
35
|
+
ResqueHelpers.arguments(payload),
|
36
|
+
Appsignal.config[:filter_parameters]
|
37
|
+
)
|
38
|
+
transaction.params = args if args
|
39
|
+
transaction.set_tags("queue" => queue)
|
40
|
+
|
41
|
+
Appsignal::Transaction.complete_current!
|
42
|
+
end
|
43
|
+
Appsignal.stop("resque")
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
class ResqueHelpers
|
49
|
+
def self.arguments(payload)
|
50
|
+
case payload["class"]
|
51
|
+
when "ActiveJob::QueueAdapters::ResqueAdapter::JobWrapper"
|
52
|
+
nil # Set in the ActiveJob integration
|
53
|
+
else
|
54
|
+
payload["args"]
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
@@ -12,7 +12,6 @@ module Appsignal
|
|
12
12
|
end
|
13
13
|
|
14
14
|
def install
|
15
|
-
require "appsignal/probes/sidekiq"
|
16
15
|
Appsignal::Minutely.probes.register :sidekiq, Appsignal::Probes::SidekiqProbe
|
17
16
|
|
18
17
|
::Sidekiq.configure_server do |config|
|
@@ -27,8 +26,7 @@ module Appsignal
|
|
27
26
|
class SidekiqPlugin # rubocop:disable Metrics/ClassLength
|
28
27
|
include Appsignal::Hooks::Helpers
|
29
28
|
|
30
|
-
|
31
|
-
JOB_KEYS = %w[
|
29
|
+
EXCLUDED_JOB_KEYS = %w[
|
32
30
|
args backtrace class created_at enqueued_at error_backtrace error_class
|
33
31
|
error_message failed_at jid retried_at retry wrapped
|
34
32
|
].freeze
|
@@ -36,7 +34,7 @@ module Appsignal
|
|
36
34
|
def call(_worker, item, _queue)
|
37
35
|
job_status = nil
|
38
36
|
transaction = Appsignal::Transaction.create(
|
39
|
-
|
37
|
+
item["jid"],
|
40
38
|
Appsignal::Transaction::BACKGROUND_JOB,
|
41
39
|
Appsignal::Transaction::GenericRequest.new(
|
42
40
|
:queue_start => item["enqueued_at"]
|
@@ -55,7 +53,10 @@ module Appsignal
|
|
55
53
|
ensure
|
56
54
|
if transaction
|
57
55
|
transaction.set_action_if_nil(formatted_action_name(item))
|
58
|
-
|
56
|
+
|
57
|
+
params = filtered_arguments(item)
|
58
|
+
transaction.params = params if params
|
59
|
+
|
59
60
|
formatted_metadata(item).each do |key, value|
|
60
61
|
transaction.set_metadata key, value
|
61
62
|
end
|
@@ -81,16 +82,20 @@ module Appsignal
|
|
81
82
|
|
82
83
|
def formatted_action_name(job)
|
83
84
|
sidekiq_action_name = parse_action_name(job)
|
85
|
+
return unless sidekiq_action_name
|
86
|
+
|
84
87
|
complete_action = sidekiq_action_name =~ /\.|#/
|
85
|
-
|
86
|
-
|
87
|
-
end
|
88
|
+
return sidekiq_action_name if complete_action
|
89
|
+
|
88
90
|
"#{sidekiq_action_name}#perform"
|
89
91
|
end
|
90
92
|
|
91
93
|
def filtered_arguments(job)
|
94
|
+
arguments = parse_arguments(job)
|
95
|
+
return unless arguments
|
96
|
+
|
92
97
|
Appsignal::Utils::HashSanitizer.sanitize(
|
93
|
-
|
98
|
+
arguments,
|
94
99
|
Appsignal.config[:filter_parameters]
|
95
100
|
)
|
96
101
|
end
|
@@ -98,7 +103,8 @@ module Appsignal
|
|
98
103
|
def formatted_metadata(item)
|
99
104
|
{}.tap do |hash|
|
100
105
|
(item || {}).each do |key, value|
|
101
|
-
next if
|
106
|
+
next if EXCLUDED_JOB_KEYS.include?(key)
|
107
|
+
|
102
108
|
hash[key] = truncate(string_or_inspect(value))
|
103
109
|
end
|
104
110
|
end
|
@@ -117,81 +123,11 @@ module Appsignal
|
|
117
123
|
safe_load(args[0], job_class) do |target, method, _|
|
118
124
|
"#{target}.#{method}"
|
119
125
|
end
|
120
|
-
when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
|
121
|
-
wrapped_job = job["wrapped"]
|
122
|
-
if wrapped_job
|
123
|
-
parse_active_job_action_name_from_wrapped job
|
124
|
-
else
|
125
|
-
parse_active_job_action_name_from_arguments job
|
126
|
-
end
|
127
126
|
else
|
128
127
|
job_class
|
129
128
|
end
|
130
129
|
end
|
131
130
|
|
132
|
-
# Return the ActiveJob wrapped job name.
|
133
|
-
#
|
134
|
-
# Returns "unknown" if no acceptable job class name could be found.
|
135
|
-
#
|
136
|
-
# @example Payload with "wrapped" value
|
137
|
-
# {
|
138
|
-
# "class" => "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper",
|
139
|
-
# "wrapped" => "MyWrappedJob",
|
140
|
-
# # ...
|
141
|
-
# }
|
142
|
-
def parse_active_job_action_name_from_wrapped(job)
|
143
|
-
job_class = job["wrapped"]
|
144
|
-
case job_class
|
145
|
-
when "ActionMailer::DeliveryJob"
|
146
|
-
extract_action_mailer_name job["args"]
|
147
|
-
when String
|
148
|
-
job_class
|
149
|
-
else
|
150
|
-
unknown_action_name_for job
|
151
|
-
end
|
152
|
-
end
|
153
|
-
|
154
|
-
# Return the ActiveJob job name based on the job's arguments.
|
155
|
-
#
|
156
|
-
# Returns "unknown" if no acceptable job class name could be found.
|
157
|
-
#
|
158
|
-
# @example Payload without "wrapped" value
|
159
|
-
# {
|
160
|
-
# "class" => "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper",
|
161
|
-
# "args" => [{
|
162
|
-
# "job_class" => "MyWrappedJob",
|
163
|
-
# # ...
|
164
|
-
# }]
|
165
|
-
# # ...
|
166
|
-
# }
|
167
|
-
def parse_active_job_action_name_from_arguments(job)
|
168
|
-
args = job.fetch("args", [])
|
169
|
-
first_arg = args[0]
|
170
|
-
if first_arg == "ActionMailer::DeliveryJob"
|
171
|
-
extract_action_mailer_name args
|
172
|
-
elsif active_job_payload?(first_arg)
|
173
|
-
first_arg["job_class"]
|
174
|
-
else
|
175
|
-
unknown_action_name_for job
|
176
|
-
end
|
177
|
-
end
|
178
|
-
|
179
|
-
# Checks if the first argument in the job payload is an ActiveJob payload.
|
180
|
-
def active_job_payload?(arg)
|
181
|
-
arg.is_a?(Hash) && arg["job_class"].is_a?(String)
|
182
|
-
end
|
183
|
-
|
184
|
-
def unknown_action_name_for(job)
|
185
|
-
Appsignal.logger.debug \
|
186
|
-
"Unable to determine an action name from Sidekiq payload: #{job}"
|
187
|
-
UNKNOWN_ACTION_NAME
|
188
|
-
end
|
189
|
-
|
190
|
-
def extract_action_mailer_name(args)
|
191
|
-
# Returns in format: MailerClass#mailer_method
|
192
|
-
args[0]["arguments"][0..1].join("#")
|
193
|
-
end
|
194
|
-
|
195
131
|
# Based on: https://github.com/mperham/sidekiq/blob/63ee43353bd3b753beb0233f64865e658abeb1c3/lib/sidekiq/api.rb#L336-L358
|
196
132
|
def parse_arguments(job)
|
197
133
|
args = job.fetch("args", [])
|
@@ -201,20 +137,7 @@ module Appsignal
|
|
201
137
|
arg
|
202
138
|
end
|
203
139
|
when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
|
204
|
-
|
205
|
-
first_arg = args[0]
|
206
|
-
job_args =
|
207
|
-
if is_wrapped || active_job_payload?(first_arg)
|
208
|
-
first_arg["arguments"]
|
209
|
-
else
|
210
|
-
[]
|
211
|
-
end
|
212
|
-
if (is_wrapped || first_arg) == "ActionMailer::DeliveryJob"
|
213
|
-
# Remove MailerClass, mailer_method and "deliver_now"
|
214
|
-
job_args.drop(3)
|
215
|
-
else
|
216
|
-
job_args
|
217
|
-
end
|
140
|
+
nil # Set in the ActiveJob integration
|
218
141
|
else
|
219
142
|
# Sidekiq Enterprise argument encryption.
|
220
143
|
# More information: https://github.com/mperham/sidekiq/wiki/Ent-Encryption
|
@@ -4,18 +4,15 @@ module Appsignal
|
|
4
4
|
module Integrations
|
5
5
|
# @api private
|
6
6
|
module ResquePlugin
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
"
|
14
|
-
:
|
15
|
-
|
16
|
-
) do
|
17
|
-
yield
|
18
|
-
end
|
7
|
+
def self.extended(_)
|
8
|
+
callers = caller
|
9
|
+
Appsignal::Utils::DeprecationMessage.message \
|
10
|
+
"The AppSignal ResquePlugin is deprecated and does " \
|
11
|
+
"nothing on extend. In this version of the AppSignal Ruby gem " \
|
12
|
+
"the integration with Resque is automatic on all Resque workers. " \
|
13
|
+
"Please remove the following line from this file to remove this " \
|
14
|
+
"message: extend Appsignal::Integrations::ResquePlugin\n" \
|
15
|
+
"#{callers.first}"
|
19
16
|
end
|
20
17
|
end
|
21
18
|
end
|