tingyun_rpm 1.4.2 → 1.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +0 -0
- data/lib/ting_yun/agent.rb +7 -1
- data/lib/ting_yun/agent/collector/error_collector.rb +103 -25
- data/lib/ting_yun/agent/collector/error_collector/error_trace_array.rb +2 -0
- data/lib/ting_yun/agent/collector/error_collector/noticed_error.rb +20 -13
- data/lib/ting_yun/agent/collector/transaction_sampler.rb +4 -1
- data/lib/ting_yun/agent/collector/transaction_sampler/class_method.rb +4 -4
- data/lib/ting_yun/agent/cross_app/cross_app_tracing.rb +1 -3
- data/lib/ting_yun/agent/instance_methods/container_data_manager.rb +6 -0
- data/lib/ting_yun/agent/method_tracer_helpers.rb +6 -4
- data/lib/ting_yun/agent/transaction.rb +15 -15
- data/lib/ting_yun/agent/transaction/apdex.rb +1 -1
- data/lib/ting_yun/agent/transaction/class_method.rb +14 -17
- data/lib/ting_yun/agent/transaction/exceptions.rb +19 -6
- data/lib/ting_yun/agent/transaction/instance_method.rb +8 -4
- data/lib/ting_yun/agent/transaction/trace.rb +23 -4
- data/lib/ting_yun/agent/transaction/trace_node.rb +26 -6
- data/lib/ting_yun/agent/transaction/traced_method_stack.rb +2 -2
- data/lib/ting_yun/agent/transaction/transaction_sample_builder.rb +17 -8
- data/lib/ting_yun/configuration/default_source.rb +22 -1
- data/lib/ting_yun/http/abstract_request.rb +23 -0
- data/lib/ting_yun/http/curb_wrappers.rb +76 -0
- data/lib/ting_yun/http/excon_wrappers.rb +81 -0
- data/lib/ting_yun/http/http_client_request.rb +2 -2
- data/lib/ting_yun/http/net_http_request.rb +2 -2
- data/lib/ting_yun/http/typhoeus_wrappers.rb +88 -0
- data/lib/ting_yun/instrumentation/bunny.rb +3 -3
- data/lib/ting_yun/instrumentation/curb.rb +191 -0
- data/lib/ting_yun/instrumentation/excon.rb +131 -0
- data/lib/ting_yun/instrumentation/grape.rb +4 -2
- data/lib/ting_yun/instrumentation/kafka.rb +3 -3
- data/lib/ting_yun/instrumentation/memcached.rb +1 -1
- data/lib/ting_yun/instrumentation/middleware_proxy.rb +17 -1
- data/lib/ting_yun/instrumentation/middleware_tracing.rb +1 -1
- data/lib/ting_yun/instrumentation/mongo_command_log_subscriber.rb +1 -0
- data/lib/ting_yun/instrumentation/net.rb +24 -24
- data/lib/ting_yun/instrumentation/rack.rb +10 -10
- data/lib/ting_yun/instrumentation/rails3/action_controller.rb +1 -1
- data/lib/ting_yun/instrumentation/rake.rb +8 -4
- data/lib/ting_yun/instrumentation/support/action_controller_subscriber.rb +1 -1
- data/lib/ting_yun/instrumentation/support/action_view_subscriber.rb +2 -2
- data/lib/ting_yun/instrumentation/support/active_record_subscriber.rb +2 -2
- data/lib/ting_yun/instrumentation/support/controller_instrumentation.rb +3 -3
- data/lib/ting_yun/instrumentation/support/external_error.rb +4 -4
- data/lib/ting_yun/instrumentation/support/external_helper.rb +6 -1
- data/lib/ting_yun/instrumentation/thrift.rb +1 -1
- data/lib/ting_yun/instrumentation/typhoeus.rb +75 -0
- data/lib/ting_yun/support/exception.rb +2 -0
- data/lib/ting_yun/support/http_clients/uri_util.rb +12 -7
- data/lib/ting_yun/ting_yun_service.rb +1 -1
- data/lib/ting_yun/ting_yun_service/upload_service.rb +15 -3
- data/lib/ting_yun/version.rb +2 -2
- metadata +9 -3
- data/lib/ting_yun/http/generic_request.rb +0 -8
@@ -10,7 +10,7 @@ module TingYun
|
|
10
10
|
attr_accessor :apdex_start, :transaction_start_time
|
11
11
|
|
12
12
|
def initialize(start, transaction_start)
|
13
|
-
@apdex_start = start || transaction_start
|
13
|
+
@apdex_start = (start || transaction_start).to_f
|
14
14
|
@transaction_start_time = transaction_start
|
15
15
|
end
|
16
16
|
|
@@ -27,13 +27,14 @@ module TingYun
|
|
27
27
|
txn = state.current_transaction
|
28
28
|
if txn
|
29
29
|
txn.exceptions.notice_error(e, options)
|
30
|
+
state.transaction_sample_builder.trace.add_errors_to_current_node(state,e) rescue nil
|
30
31
|
elsif TingYun::Agent.instance
|
31
32
|
TingYun::Agent.instance.error_collector.notice_error(e, options)
|
32
33
|
end
|
33
34
|
end
|
34
35
|
|
35
36
|
|
36
|
-
def stop(state, end_time = Time.now, summary_metric_names=[])
|
37
|
+
def stop(state, end_time = Time.now.to_f, summary_metric_names=[])
|
37
38
|
|
38
39
|
txn = state.current_transaction
|
39
40
|
|
@@ -48,23 +49,23 @@ module TingYun
|
|
48
49
|
txn.stop(state, end_time, nested_frame, summary_metric_names)
|
49
50
|
state.reset
|
50
51
|
else
|
51
|
-
nested_name = nested_transaction_name
|
52
|
+
nested_name = Transaction.nested_transaction_name nested_frame.name
|
52
53
|
|
53
|
-
if nested_name.start_with?(MIDDLEWARE_PREFIX)
|
54
|
-
|
55
|
-
else
|
56
|
-
|
57
|
-
end
|
58
|
-
summary_metrics = summary_metric_names unless summary_metric_names.empty?
|
54
|
+
# if nested_name.start_with?(MIDDLEWARE_PREFIX)
|
55
|
+
# summary_metrics = MIDDLEWARE_SUMMARY_METRICS
|
56
|
+
# else
|
57
|
+
# summary_metrics = EMPTY_SUMMARY_METRICS
|
58
|
+
# end
|
59
|
+
# summary_metrics = summary_metric_names unless summary_metric_names.empty?
|
59
60
|
|
60
61
|
TingYun::Agent::MethodTracerHelpers.trace_execution_scoped_footer(
|
61
62
|
state,
|
62
|
-
nested_frame.start_time
|
63
|
+
nested_frame.start_time,
|
63
64
|
nested_name,
|
64
|
-
|
65
|
+
EMPTY_SUMMARY_METRICS,
|
65
66
|
nested_frame,
|
66
67
|
NESTED_TRACE_STOP_OPTIONS,
|
67
|
-
end_time
|
68
|
+
end_time)
|
68
69
|
|
69
70
|
end
|
70
71
|
|
@@ -83,11 +84,11 @@ module TingYun
|
|
83
84
|
# to be absolutely sure we don't report agent problems as app errors
|
84
85
|
yield
|
85
86
|
rescue => e
|
86
|
-
|
87
|
+
::TingYun::Agent.notice_error(e,:type=> :exception)
|
87
88
|
raise e
|
88
89
|
ensure
|
89
90
|
# when kafka consumer in task, drop original web_action
|
90
|
-
Transaction.stop(state, Time.now, summary_metrics) if state.current_transaction
|
91
|
+
Transaction.stop(state, Time.now.to_f, summary_metrics) if state.current_transaction
|
91
92
|
end
|
92
93
|
end
|
93
94
|
|
@@ -100,10 +101,6 @@ module TingYun
|
|
100
101
|
else
|
101
102
|
txn = start_new_transaction(state, category, options)
|
102
103
|
end
|
103
|
-
|
104
|
-
# merge params every step into here
|
105
|
-
txn.attributes.merge_request_parameters(options[:filtered_params])
|
106
|
-
|
107
104
|
txn
|
108
105
|
rescue => e
|
109
106
|
TingYun::Agent.logger.error("Exception during Transaction.start", e)
|
@@ -6,6 +6,7 @@ module TingYun
|
|
6
6
|
class Exceptions
|
7
7
|
attr_accessor :exceptions
|
8
8
|
|
9
|
+
|
9
10
|
def initialize
|
10
11
|
@exceptions = {}
|
11
12
|
end
|
@@ -20,7 +21,7 @@ module TingYun
|
|
20
21
|
end
|
21
22
|
|
22
23
|
# Do not call this. Invoke the class method instead.
|
23
|
-
def notice_error(error, options={}) # :nodoc:
|
24
|
+
def notice_error(error, options={}) # :nodoc:11
|
24
25
|
if @exceptions[error]
|
25
26
|
@exceptions[error].merge! options
|
26
27
|
else
|
@@ -30,12 +31,24 @@ module TingYun
|
|
30
31
|
|
31
32
|
#collector error
|
32
33
|
def had_error?
|
33
|
-
|
34
|
-
return false
|
35
|
-
else
|
36
|
-
return true
|
37
|
-
end
|
34
|
+
@have ||= count_errors == 0? false : true
|
38
35
|
end
|
36
|
+
|
37
|
+
def errors_and_exceptions
|
38
|
+
[count_errors, exceptions.size - count_errors]
|
39
|
+
end
|
40
|
+
|
41
|
+
def count_errors
|
42
|
+
@count_errors ||= errors.size
|
43
|
+
end
|
44
|
+
|
45
|
+
def errors
|
46
|
+
@errors ||= exceptions.select{|k,v| v[:type]==:error}
|
47
|
+
end
|
48
|
+
def exceptions
|
49
|
+
@exceptions ||= exceptions.select{|k,v| v[:type]==:exception}
|
50
|
+
end
|
51
|
+
|
39
52
|
end
|
40
53
|
end
|
41
54
|
end
|
@@ -14,6 +14,10 @@ module TingYun
|
|
14
14
|
end
|
15
15
|
|
16
16
|
def create_nested_frame(state, category, options)
|
17
|
+
|
18
|
+
if options[:filtered_params] && !options[:filtered_params].empty?
|
19
|
+
attributes.merge_request_parameters(options[:filtered_params])
|
20
|
+
end
|
17
21
|
@has_children = true
|
18
22
|
frame_stack.push TingYun::Agent::MethodTracerHelpers.trace_execution_scoped_header(state, Time.now.to_f)
|
19
23
|
name_last_frame(options[:transaction_name])
|
@@ -71,7 +75,7 @@ module TingYun
|
|
71
75
|
|
72
76
|
def record_summary_metrics(state, outermost_node_name,end_time)
|
73
77
|
unless @frozen_name == outermost_node_name
|
74
|
-
time = (end_time
|
78
|
+
time = (end_time - start_time) * 1000
|
75
79
|
@metrics.record_unscoped(@frozen_name, time)
|
76
80
|
if @frozen_name.start_with?('WebAction')
|
77
81
|
state.current_transaction.base_quantile_hash[@frozen_name] = time
|
@@ -79,7 +83,7 @@ module TingYun
|
|
79
83
|
end
|
80
84
|
end
|
81
85
|
|
82
|
-
def assign_agent_attributes
|
86
|
+
def assign_agent_attributes(state)
|
83
87
|
|
84
88
|
@attributes.add_agent_attribute(:threadName, "pid-#{$$}");
|
85
89
|
|
@@ -89,7 +93,7 @@ module TingYun
|
|
89
93
|
|
90
94
|
@attributes.add_agent_attribute(:tx_id, @guid);
|
91
95
|
@attributes.add_agent_attribute(:metric_name, best_name);
|
92
|
-
|
96
|
+
@attributes.add_agent_attribute(:trace_id, state.trace_id || "0")
|
93
97
|
end
|
94
98
|
|
95
99
|
|
@@ -131,7 +135,7 @@ module TingYun
|
|
131
135
|
unless @frozen_name
|
132
136
|
@frozen_name = best_name
|
133
137
|
end
|
134
|
-
|
138
|
+
@frozen_name = CONTROLLER_PREFIX + @frozen_name unless @frozen_name.start_with? CONTROLLER_PREFIX,BACKGROUND_PREFIX
|
135
139
|
yield if block_given?
|
136
140
|
end
|
137
141
|
|
@@ -1,14 +1,15 @@
|
|
1
|
-
|
1
|
+
# encoding: utf-8
|
2
2
|
require 'ting_yun/agent/transaction/trace_node'
|
3
3
|
require 'ting_yun/support/helper'
|
4
4
|
require 'ting_yun/support/coerce'
|
5
5
|
require 'ting_yun/agent/database'
|
6
|
+
require 'set'
|
6
7
|
module TingYun
|
7
8
|
module Agent
|
8
9
|
class Transaction
|
9
10
|
class Trace
|
10
11
|
|
11
|
-
attr_accessor :root_node, :node_count, :threshold, :guid, :attributes, :start_time, :finished
|
12
|
+
attr_accessor :root_node, :node_count, :threshold, :guid, :attributes, :start_time, :finished, :array_size,:e_set
|
12
13
|
|
13
14
|
def initialize(start_time)
|
14
15
|
@start_time = start_time
|
@@ -16,6 +17,7 @@ module TingYun
|
|
16
17
|
@prepared = false
|
17
18
|
@guid = generate_guid
|
18
19
|
@root_node = TingYun::Agent::Transaction::TraceNode.new(0.0, "ROOT")
|
20
|
+
@e_set = Set.new
|
19
21
|
end
|
20
22
|
|
21
23
|
def create_node(time_since_start, metric_name = nil)
|
@@ -31,7 +33,7 @@ module TingYun
|
|
31
33
|
EMPTY_STRING = ''.freeze
|
32
34
|
|
33
35
|
include TingYun::Support::Coerce
|
34
|
-
|
36
|
+
|
35
37
|
def trace_tree
|
36
38
|
[
|
37
39
|
@start_time.round,
|
@@ -50,7 +52,7 @@ module TingYun
|
|
50
52
|
encoder.encode(trace_tree),
|
51
53
|
attributes.agent_attributes[:tx_id],
|
52
54
|
guid
|
53
|
-
]
|
55
|
+
] + array_size
|
54
56
|
end
|
55
57
|
|
56
58
|
def prepare_to_send!
|
@@ -110,6 +112,23 @@ module TingYun
|
|
110
112
|
attributes.request_params
|
111
113
|
end
|
112
114
|
|
115
|
+
def add_errors(errors)
|
116
|
+
errors.each do |error|
|
117
|
+
unless @e_set.member? error.object_id
|
118
|
+
@e_set.add error.object_id
|
119
|
+
root_node.add_error(error)
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
def add_errors_to_current_node(state, error)
|
125
|
+
unless @e_set.member? error.object_id
|
126
|
+
@e_set.add error.object_id
|
127
|
+
state.transaction_sample_builder.current_node.add_error(error)
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
131
|
+
|
113
132
|
HEX_DIGITS = (0..15).map{|i| i.to_s(16)}
|
114
133
|
GUID_LENGTH = 16
|
115
134
|
|
@@ -22,6 +22,7 @@ module TingYun
|
|
22
22
|
@metric_name = metric_name || UNKNOWN_NODE_NAME
|
23
23
|
@called_nodes = nil
|
24
24
|
@count = 1
|
25
|
+
self["exception"] ||= []
|
25
26
|
end
|
26
27
|
|
27
28
|
def add_called_node(s)
|
@@ -41,11 +42,11 @@ module TingYun
|
|
41
42
|
|
42
43
|
|
43
44
|
def pre_metric_name(metric_name)
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
45
|
+
@name ||= if metric_name.start_with?('Database ')
|
46
|
+
"#{metric_name.split('/')[0]}%2F#{metric_name.split('%2F')[-1]}"
|
47
|
+
else
|
48
|
+
metric_name
|
49
|
+
end
|
49
50
|
end
|
50
51
|
|
51
52
|
def to_array
|
@@ -57,7 +58,7 @@ module TingYun
|
|
57
58
|
TingYun::Support::Coerce.string(klass)||TingYun::Support::Coerce.string(pre_metric_name(metric_name)),
|
58
59
|
TingYun::Support::Coerce.string(method)||'',
|
59
60
|
params] +
|
60
|
-
|
61
|
+
[(@called_nodes ? @called_nodes.map{|s| s.to_array} : [])]
|
61
62
|
end
|
62
63
|
|
63
64
|
def custom_params
|
@@ -110,6 +111,25 @@ module TingYun
|
|
110
111
|
TingYun::Agent::Database.explain_sql(statement)
|
111
112
|
end
|
112
113
|
|
114
|
+
def add_error(error)
|
115
|
+
if error.respond_to?(:tingyun_external)
|
116
|
+
self["exception"] << {"message" => error.message,
|
117
|
+
"class" => "External #{error.tingyun_code}"
|
118
|
+
}
|
119
|
+
else
|
120
|
+
if ::TingYun::Agent.config[:'nbs.exception.stack_enabled']
|
121
|
+
self["exception"] << {"message" => error.message,
|
122
|
+
"class" => error.class.name ,
|
123
|
+
"stacktrace"=> error.backtrace.reject! { |t| t.include?('tingyun_rpm') }
|
124
|
+
}
|
125
|
+
else
|
126
|
+
self["exception"] << {"message" => error.message,
|
127
|
+
"class" => error.class.name
|
128
|
+
}
|
129
|
+
end
|
130
|
+
end
|
131
|
+
end
|
132
|
+
|
113
133
|
protected
|
114
134
|
def parent_node=(s)
|
115
135
|
@parent_node = s
|
@@ -25,10 +25,10 @@ module TingYun
|
|
25
25
|
frame
|
26
26
|
end
|
27
27
|
|
28
|
-
def pop_frame(state, expected_frame, name, time, deduct_call_time_from_parent=true, klass_name=nil)
|
28
|
+
def pop_frame(state, expected_frame, name, time, deduct_call_time_from_parent=true, klass_name=nil, error = nil)
|
29
29
|
frame = fetch_matching_frame(expected_frame)
|
30
30
|
note_children_time(frame, time, deduct_call_time_from_parent)
|
31
|
-
transaction_sampler.notice_pop_frame(state, name, time, klass_name) if sampler_enabled?
|
31
|
+
transaction_sampler.notice_pop_frame(state, name, time, klass_name, error) if sampler_enabled?
|
32
32
|
frame.name = name
|
33
33
|
frame
|
34
34
|
end
|
@@ -5,7 +5,7 @@ require 'ting_yun/agent/transaction/trace'
|
|
5
5
|
|
6
6
|
module TingYun
|
7
7
|
module Agent
|
8
|
-
class
|
8
|
+
class TransactionSampleBuilder
|
9
9
|
|
10
10
|
class PlaceholderNode
|
11
11
|
attr_reader :parent_node
|
@@ -28,21 +28,21 @@ module TingYun
|
|
28
28
|
|
29
29
|
attr_reader :current_node, :trace
|
30
30
|
|
31
|
-
def initialize(time=Time.now)
|
32
|
-
@trace = TingYun::Agent::Transaction::Trace.new(time
|
33
|
-
@trace_start = time
|
31
|
+
def initialize(time=Time.now.to_f)
|
32
|
+
@trace = TingYun::Agent::Transaction::Trace.new(time)
|
33
|
+
@trace_start = time
|
34
34
|
@current_node = @trace.root_node
|
35
35
|
end
|
36
36
|
|
37
37
|
def trace_entry(time)
|
38
38
|
if @trace.node_count == 0
|
39
|
-
node = @trace.create_node(time
|
39
|
+
node = @trace.create_node(time - @trace_start)
|
40
40
|
@trace.root_node = node
|
41
41
|
@current_node = node
|
42
42
|
return @current_node
|
43
43
|
end
|
44
44
|
if @trace.node_count < node_limit
|
45
|
-
node = @trace.create_node(time
|
45
|
+
node = @trace.create_node(time - @trace_start)
|
46
46
|
@current_node.add_called_node(node)
|
47
47
|
@current_node = node
|
48
48
|
|
@@ -59,7 +59,7 @@ module TingYun
|
|
59
59
|
@current_node
|
60
60
|
end
|
61
61
|
|
62
|
-
def trace_exit(metric_name, time, klass_name)
|
62
|
+
def trace_exit(metric_name, time, klass_name, error = nil)
|
63
63
|
if @current_node.is_a?(PlaceholderNode)
|
64
64
|
@current_node.depth -= 1
|
65
65
|
if @current_node.depth == 0
|
@@ -68,9 +68,18 @@ module TingYun
|
|
68
68
|
else
|
69
69
|
@current_node.metric_name = metric_name
|
70
70
|
@current_node.klass = klass_name
|
71
|
-
@current_node.end_trace(time
|
71
|
+
@current_node.end_trace(time - @trace_start)
|
72
72
|
@current_node = @current_node.parent_node
|
73
73
|
end
|
74
|
+
if error
|
75
|
+
unless trace.e_set.member? error.object_id
|
76
|
+
trace.e_set.add error.object_id
|
77
|
+
@current_node["exception"] << {"message" => error.message,
|
78
|
+
"class" => error.class.to_s,
|
79
|
+
"stacktrace"=> error.backtrace
|
80
|
+
}
|
81
|
+
end
|
82
|
+
end
|
74
83
|
end
|
75
84
|
|
76
85
|
def finish_trace(time=Time.now.to_f)
|
@@ -634,12 +634,19 @@ module TingYun
|
|
634
634
|
:allowed_from_server => true,
|
635
635
|
:description => 'trace ID of crossing apps'
|
636
636
|
},
|
637
|
+
:'naming.rules_enabled' => {
|
638
|
+
:default => false,
|
639
|
+
:public => true,
|
640
|
+
:type => Boolean,
|
641
|
+
:allowed_from_server => false,
|
642
|
+
:description => 'Enable or disable of the naming.rules function'
|
643
|
+
},
|
637
644
|
:'nbs.naming.rules' => {
|
638
645
|
:default => "[]",
|
639
646
|
:public => true,
|
640
647
|
:type => String,
|
641
648
|
:allowed_from_server => true,
|
642
|
-
:description => 'defined
|
649
|
+
:description => 'defined name rule '
|
643
650
|
},
|
644
651
|
:disable_rake => {
|
645
652
|
:default => true,
|
@@ -655,6 +662,13 @@ module TingYun
|
|
655
662
|
:allowed_from_server => false,
|
656
663
|
:description => 'Specify an array of Rake tasks to automatically instrument.'
|
657
664
|
},
|
665
|
+
:'rake.black.tasks' => {
|
666
|
+
:default => [],
|
667
|
+
:public => true,
|
668
|
+
:type => Array,
|
669
|
+
:allowed_from_server => false,
|
670
|
+
:description => 'Specify an array of Rake tasks to automatically uninstrument.'
|
671
|
+
},
|
658
672
|
:'nbs.transaction_tracer.thrift' =>{
|
659
673
|
:default => true,
|
660
674
|
:public => true,
|
@@ -682,6 +696,13 @@ module TingYun
|
|
682
696
|
:type => Boolean,
|
683
697
|
:allowed_from_server => true,
|
684
698
|
:description => 'Enable or disable the mq feature'
|
699
|
+
},
|
700
|
+
:'nbs.exception.stack_enabled' => {
|
701
|
+
:default => false,
|
702
|
+
:public => true,
|
703
|
+
:type => Boolean,
|
704
|
+
:allowed_from_server => true,
|
705
|
+
:description => 'Enable or disable the exception trace'
|
685
706
|
}
|
686
707
|
}.freeze
|
687
708
|
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require 'ting_yun/support/http_clients/uri_util'
|
3
|
+
module TingYun
|
4
|
+
module Http
|
5
|
+
class AbstractRequest
|
6
|
+
ERROR_MESSAGE = 'Subclasses of TingYun::Http::AbstractRequest must implement a '.freeze
|
7
|
+
|
8
|
+
def []
|
9
|
+
raise NotImplementedError, ERROR_MESSAGE + ':[] method'
|
10
|
+
end
|
11
|
+
|
12
|
+
def []=
|
13
|
+
raise NotImplementedError, ERROR_MESSAGE + ':[]= method'
|
14
|
+
end
|
15
|
+
|
16
|
+
def from
|
17
|
+
raise NotImplementedError, ERROR_MESSAGE + ':from method'
|
18
|
+
end
|
19
|
+
|
20
|
+
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|