ddtrace 0.29.1 → 0.30.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. checksums.yaml +5 -5
  2. data/.rubocop.yml +4 -0
  3. data/CHANGELOG.md +18 -1
  4. data/lib/ddtrace/context.rb +26 -10
  5. data/lib/ddtrace/contrib/action_pack/action_controller/patcher.rb +3 -15
  6. data/lib/ddtrace/contrib/action_pack/patcher.rb +3 -9
  7. data/lib/ddtrace/contrib/action_view/event.rb +39 -0
  8. data/lib/ddtrace/contrib/action_view/events.rb +30 -0
  9. data/lib/ddtrace/contrib/action_view/events/render_partial.rb +40 -0
  10. data/lib/ddtrace/contrib/action_view/events/render_template.rb +43 -0
  11. data/lib/ddtrace/contrib/action_view/instrumentation/partial_renderer.rb +4 -12
  12. data/lib/ddtrace/contrib/action_view/instrumentation/template_renderer.rb +6 -14
  13. data/lib/ddtrace/contrib/action_view/patcher.rb +19 -25
  14. data/lib/ddtrace/contrib/active_model_serializers/patcher.rb +3 -10
  15. data/lib/ddtrace/contrib/active_record/patcher.rb +3 -9
  16. data/lib/ddtrace/contrib/active_support/cache/patcher.rb +10 -24
  17. data/lib/ddtrace/contrib/active_support/patcher.rb +3 -9
  18. data/lib/ddtrace/contrib/aws/patcher.rb +7 -13
  19. data/lib/ddtrace/contrib/concurrent_ruby/patcher.rb +4 -11
  20. data/lib/ddtrace/contrib/dalli/patcher.rb +4 -10
  21. data/lib/ddtrace/contrib/delayed_job/patcher.rb +4 -10
  22. data/lib/ddtrace/contrib/elasticsearch/patcher.rb +8 -14
  23. data/lib/ddtrace/contrib/ethon/patcher.rb +7 -9
  24. data/lib/ddtrace/contrib/excon/patcher.rb +4 -11
  25. data/lib/ddtrace/contrib/faraday/patcher.rb +6 -12
  26. data/lib/ddtrace/contrib/grape/patcher.rb +7 -13
  27. data/lib/ddtrace/contrib/graphql/patcher.rb +5 -11
  28. data/lib/ddtrace/contrib/grpc/patcher.rb +7 -13
  29. data/lib/ddtrace/contrib/http/patcher.rb +3 -9
  30. data/lib/ddtrace/contrib/mongodb/patcher.rb +5 -11
  31. data/lib/ddtrace/contrib/mysql2/patcher.rb +3 -9
  32. data/lib/ddtrace/contrib/patcher.rb +38 -10
  33. data/lib/ddtrace/contrib/racecar/patcher.rb +4 -10
  34. data/lib/ddtrace/contrib/rack/patcher.rb +56 -21
  35. data/lib/ddtrace/contrib/rails/patcher.rb +4 -8
  36. data/lib/ddtrace/contrib/rake/patcher.rb +4 -10
  37. data/lib/ddtrace/contrib/redis/patcher.rb +8 -14
  38. data/lib/ddtrace/contrib/resque/patcher.rb +4 -10
  39. data/lib/ddtrace/contrib/rest_client/patcher.rb +5 -7
  40. data/lib/ddtrace/contrib/sequel/patcher.rb +4 -10
  41. data/lib/ddtrace/contrib/shoryuken/patcher.rb +4 -10
  42. data/lib/ddtrace/contrib/sidekiq/patcher.rb +12 -18
  43. data/lib/ddtrace/contrib/sinatra/patcher.rb +4 -10
  44. data/lib/ddtrace/contrib/sucker_punch/patcher.rb +7 -13
  45. data/lib/ddtrace/diagnostics/health.rb +9 -2
  46. data/lib/ddtrace/ext/diagnostics.rb +6 -0
  47. data/lib/ddtrace/ext/sampling.rb +13 -0
  48. data/lib/ddtrace/sampler.rb +49 -8
  49. data/lib/ddtrace/sampling.rb +2 -0
  50. data/lib/ddtrace/sampling/matcher.rb +57 -0
  51. data/lib/ddtrace/sampling/rate_limiter.rb +127 -0
  52. data/lib/ddtrace/sampling/rule.rb +61 -0
  53. data/lib/ddtrace/sampling/rule_sampler.rb +111 -0
  54. data/lib/ddtrace/span.rb +12 -0
  55. data/lib/ddtrace/tracer.rb +1 -0
  56. data/lib/ddtrace/version.rb +2 -2
  57. metadata +27 -4
@@ -9,19 +9,13 @@ module Datadog
9
9
 
10
10
  module_function
11
11
 
12
- def patched?
13
- done?(:sinatra)
12
+ def target_version
13
+ Integration.version
14
14
  end
15
15
 
16
16
  def patch
17
- do_once(:sinatra) do
18
- begin
19
- require 'ddtrace/contrib/sinatra/tracer'
20
- register_tracer
21
- rescue StandardError => e
22
- Datadog::Tracer.log.error("Unable to apply Sinatra integration: #{e}")
23
- end
24
- end
17
+ require 'ddtrace/contrib/sinatra/tracer'
18
+ register_tracer
25
19
  end
26
20
 
27
21
  def register_tracer
@@ -11,23 +11,17 @@ module Datadog
11
11
 
12
12
  module_function
13
13
 
14
- def patched?
15
- done?(:sucker_punch)
14
+ def target_version
15
+ Integration.version
16
16
  end
17
17
 
18
18
  def patch
19
- do_once(:sucker_punch) do
20
- begin
21
- require 'ddtrace/contrib/sucker_punch/exception_handler'
22
- require 'ddtrace/contrib/sucker_punch/instrumentation'
19
+ require 'ddtrace/contrib/sucker_punch/exception_handler'
20
+ require 'ddtrace/contrib/sucker_punch/instrumentation'
23
21
 
24
- add_pin!
25
- ExceptionHandler.patch!
26
- Instrumentation.patch!
27
- rescue StandardError => e
28
- Datadog::Tracer.log.error("Unable to apply SuckerPunch integration: #{e}")
29
- end
30
- end
22
+ add_pin!
23
+ ExceptionHandler.patch!
24
+ Instrumentation.patch!
31
25
  end
32
26
 
33
27
  def add_pin!
@@ -10,14 +10,21 @@ module Datadog
10
10
  count :api_errors, Ext::Diagnostics::Health::Metrics::METRIC_API_ERRORS
11
11
  count :api_requests, Ext::Diagnostics::Health::Metrics::METRIC_API_REQUESTS
12
12
  count :api_responses, Ext::Diagnostics::Health::Metrics::METRIC_API_RESPONSES
13
+ count :error_context_overflow, Ext::Diagnostics::Health::Metrics::METRIC_ERROR_CONTEXT_OVERFLOW
14
+ count :error_instrumentation_patch, Ext::Diagnostics::Health::Metrics::METRIC_ERROR_INSTRUMENTATION_PATCH
15
+ count :error_span_finish, Ext::Diagnostics::Health::Metrics::METRIC_ERROR_SPAN_FINISH
16
+ count :error_unfinished_spans, Ext::Diagnostics::Health::Metrics::METRIC_ERROR_UNFINISHED_SPANS
17
+ count :instrumentation_patched, Ext::Diagnostics::Health::Metrics::METRIC_INSTRUMENTATION_PATCHED
13
18
  count :queue_accepted, Ext::Diagnostics::Health::Metrics::METRIC_QUEUE_ACCEPTED
14
19
  count :queue_accepted_lengths, Ext::Diagnostics::Health::Metrics::METRIC_QUEUE_ACCEPTED_LENGTHS
15
20
  count :queue_dropped, Ext::Diagnostics::Health::Metrics::METRIC_QUEUE_DROPPED
21
+ count :traces_filtered, Ext::Diagnostics::Health::Metrics::METRIC_TRACES_FILTERED
22
+ count :writer_cpu_time, Ext::Diagnostics::Health::Metrics::METRIC_WRITER_CPU_TIME
23
+
16
24
  gauge :queue_length, Ext::Diagnostics::Health::Metrics::METRIC_QUEUE_LENGTH
17
25
  gauge :queue_max_length, Ext::Diagnostics::Health::Metrics::METRIC_QUEUE_MAX_LENGTH
18
26
  gauge :queue_spans, Ext::Diagnostics::Health::Metrics::METRIC_QUEUE_SPANS
19
- count :traces_filtered, Ext::Diagnostics::Health::Metrics::METRIC_TRACES_FILTERED
20
- count :writer_cpu_time, Ext::Diagnostics::Health::Metrics::METRIC_WRITER_CPU_TIME
27
+ gauge :sampling_service_cache_length, Ext::Diagnostics::Health::Metrics::METRIC_SAMPLING_SERVICE_CACHE_LENGTH
21
28
  end
22
29
 
23
30
  module_function
@@ -10,12 +10,18 @@ module Datadog
10
10
  METRIC_API_ERRORS = 'datadog.tracer.api.errors'.freeze
11
11
  METRIC_API_REQUESTS = 'datadog.tracer.api.requests'.freeze
12
12
  METRIC_API_RESPONSES = 'datadog.tracer.api.responses'.freeze
13
+ METRIC_ERROR_CONTEXT_OVERFLOW = 'datadog.tracer.error.context_overflow'.freeze
14
+ METRIC_ERROR_INSTRUMENTATION_PATCH = 'datadog.tracer.error.instrumentation_patch'.freeze
15
+ METRIC_ERROR_SPAN_FINISH = 'datadog.tracer.error.span_finish'.freeze
16
+ METRIC_ERROR_UNFINISHED_SPANS = 'datadog.tracer.error.unfinished_spans'.freeze
17
+ METRIC_INSTRUMENTATION_PATCHED = 'datadog.tracer.instrumentation_patched'.freeze
13
18
  METRIC_QUEUE_ACCEPTED = 'datadog.tracer.queue.accepted'.freeze
14
19
  METRIC_QUEUE_ACCEPTED_LENGTHS = 'datadog.tracer.queue.accepted_lengths'.freeze
15
20
  METRIC_QUEUE_DROPPED = 'datadog.tracer.queue.dropped'.freeze
16
21
  METRIC_QUEUE_LENGTH = 'datadog.tracer.queue.length'.freeze
17
22
  METRIC_QUEUE_MAX_LENGTH = 'datadog.tracer.queue.max_length'.freeze
18
23
  METRIC_QUEUE_SPANS = 'datadog.tracer.queue.spans'.freeze
24
+ METRIC_SAMPLING_SERVICE_CACHE_LENGTH = 'datadog.tracer.sampling.service_cache_length'.freeze
19
25
  METRIC_TRACES_FILTERED = 'datadog.tracer.traces.filtered'.freeze
20
26
  METRIC_WRITER_CPU_TIME = 'datadog.tracer.writer.cpu_time'.freeze
21
27
  end
@@ -0,0 +1,13 @@
1
+ module Datadog
2
+ module Ext
3
+ module Sampling
4
+ # If rule sampling is applied to a span, set this metric the sample rate configured for that rule.
5
+ # This should be done regardless of sampling outcome.
6
+ RULE_SAMPLE_RATE = '_dd.rule_psr'.freeze
7
+
8
+ # If rate limiting is checked on a span, set this metric the effective rate limiting rate applied.
9
+ # This should be done regardless of rate limiting outcome.
10
+ RATE_LIMITER_RATE = '_dd.limit_psr'.freeze
11
+ end
12
+ end
13
+ end
@@ -1,6 +1,7 @@
1
1
  require 'forwardable'
2
2
 
3
3
  require 'ddtrace/ext/priority'
4
+ require 'ddtrace/diagnostics/health'
4
5
 
5
6
  module Datadog
6
7
  # \Sampler performs client-side trace sampling.
@@ -12,6 +13,10 @@ module Datadog
12
13
  def sample!(_span)
13
14
  raise NotImplementedError, 'Samplers must implement the #sample! method'
14
15
  end
16
+
17
+ def sample_rate(span)
18
+ raise NotImplementedError, 'Samplers must implement the #sample_rate method'
19
+ end
15
20
  end
16
21
 
17
22
  # \AllSampler samples all the traces.
@@ -23,6 +28,10 @@ module Datadog
23
28
  def sample!(span)
24
29
  span.sampled = true
25
30
  end
31
+
32
+ def sample_rate(*_)
33
+ 1.0
34
+ end
26
35
  end
27
36
 
28
37
  # \RateSampler is based on a sample rate.
@@ -30,8 +39,6 @@ module Datadog
30
39
  KNUTH_FACTOR = 1111111111111111111
31
40
  SAMPLE_RATE_METRIC_KEY = '_sample_rate'.freeze
32
41
 
33
- attr_reader :sample_rate
34
-
35
42
  # Initialize a \RateSampler.
36
43
  # This sampler keeps a random subset of the traces. Its main purpose is to
37
44
  # reduce the instrumentation footprint.
@@ -48,6 +55,10 @@ module Datadog
48
55
  self.sample_rate = sample_rate
49
56
  end
50
57
 
58
+ def sample_rate(*_)
59
+ @sample_rate
60
+ end
61
+
51
62
  def sample_rate=(sample_rate)
52
63
  @sample_rate = sample_rate
53
64
  @sampling_id_threshold = sample_rate * Span::MAX_ID
@@ -136,6 +147,10 @@ module Datadog
136
147
  end
137
148
  end
138
149
 
150
+ def length
151
+ @samplers.length
152
+ end
153
+
139
154
  private
140
155
 
141
156
  def set_rate(key, rate)
@@ -159,6 +174,9 @@ module Datadog
159
174
 
160
175
  # Update each service rate
161
176
  update_all(rate_by_service)
177
+
178
+ # Emit metric for service cache size
179
+ Diagnostics::Health.metrics.sampling_service_cache_length(length)
162
180
  end
163
181
 
164
182
  private
@@ -195,10 +213,8 @@ module Datadog
195
213
  # If priority sampling has already been applied upstream, use that, otherwise...
196
214
  unless priority_assigned_upstream?(span)
197
215
  # Roll the dice and determine whether how we set the priority.
198
- # NOTE: We'll want to leave `span.sampled = true` here; all spans for priority sampling must
199
- # be sent to the agent. Otherwise metrics for traces will not be accurate, since the
200
- # agent will have an incomplete dataset.
201
- priority = priority_sample(span) ? Datadog::Ext::Priority::AUTO_KEEP : Datadog::Ext::Priority::AUTO_REJECT
216
+ priority = priority_sample!(span) ? Datadog::Ext::Priority::AUTO_KEEP : Datadog::Ext::Priority::AUTO_REJECT
217
+
202
218
  assign_priority!(span, priority)
203
219
  end
204
220
  else
@@ -229,8 +245,33 @@ module Datadog
229
245
  span.context && !span.context.sampling_priority.nil?
230
246
  end
231
247
 
232
- def priority_sample(span)
233
- @priority_sampler.sample?(span)
248
+ def priority_sample!(span)
249
+ preserving_sampling(span) do
250
+ @priority_sampler.sample!(span)
251
+ end
252
+ end
253
+
254
+ # Ensures the span is always propagated to the writer and that
255
+ # the sample rate metric represents the true client-side sampling.
256
+ def preserving_sampling(span)
257
+ pre_sample_rate_metric = span.get_metric(SAMPLE_RATE_METRIC_KEY)
258
+
259
+ yield.tap do
260
+ # NOTE: We'll want to leave `span.sampled = true` here; all spans for priority sampling must
261
+ # be sent to the agent. Otherwise metrics for traces will not be accurate, since the
262
+ # agent will have an incomplete dataset.
263
+ #
264
+ # We also ensure that the agent knows we that our `post_sampler` is not performing true sampling,
265
+ # to avoid erroneous metric upscaling.
266
+ span.sampled = true
267
+ if pre_sample_rate_metric
268
+ # Restore true sampling metric, as only the @pre_sampler can reject traces
269
+ span.set_metric(SAMPLE_RATE_METRIC_KEY, pre_sample_rate_metric)
270
+ else
271
+ # If @pre_sampler is not enable, sending this metric would be misleading
272
+ span.clear_metric(SAMPLE_RATE_METRIC_KEY)
273
+ end
274
+ end
234
275
  end
235
276
 
236
277
  def assign_priority!(span, priority)
@@ -0,0 +1,2 @@
1
+ require 'ddtrace/sampling/rule'
2
+ require 'ddtrace/sampling/rule_sampler'
@@ -0,0 +1,57 @@
1
+ module Datadog
2
+ module Sampling
3
+ # Checks if a span conforms to a matching criteria.
4
+ class Matcher
5
+ # Returns `true` if the span should conforms to this rule, `false` otherwise
6
+ #
7
+ # @abstract
8
+ # @param [Span] span
9
+ # @return [Boolean]
10
+ def match?(span)
11
+ raise NotImplementedError
12
+ end
13
+ end
14
+
15
+ # A \Matcher that supports matching a span by
16
+ # operation name and/or service name.
17
+ class SimpleMatcher < Matcher
18
+ # Returns `true` for case equality (===) with any object
19
+ MATCH_ALL = Class.new do
20
+ # DEV: A class that implements `#===` is ~20% faster than
21
+ # DEV: a `Proc` that always returns `true`.
22
+ def ===(other)
23
+ true
24
+ end
25
+ end.new
26
+
27
+ attr_reader :name, :service
28
+
29
+ # @param name [String,Regexp,Proc] Matcher for case equality (===) with the span name, defaults to always match
30
+ # @param service [String,Regexp,Proc] Matcher for case equality (===) with the service name, defaults to always match
31
+ def initialize(name: MATCH_ALL, service: MATCH_ALL)
32
+ @name = name
33
+ @service = service
34
+ end
35
+
36
+ def match?(span)
37
+ name === span.name && service === span.service
38
+ end
39
+ end
40
+
41
+ # A \Matcher that allows for arbitrary span matching
42
+ # based on the return value of a provided block.
43
+ class ProcMatcher < Matcher
44
+ attr_reader :block
45
+
46
+ # @yield [name, service] Provides span name and service to the block
47
+ # @yieldreturn [Boolean] Whether the span conforms to this matcher
48
+ def initialize(&block)
49
+ @block = block
50
+ end
51
+
52
+ def match?(span)
53
+ block.call(span.name, span.service)
54
+ end
55
+ end
56
+ end
57
+ end
@@ -0,0 +1,127 @@
1
+ require 'ddtrace/utils/time'
2
+
3
+ module Datadog
4
+ module Sampling
5
+ # Checks for rate limiting on a resource.
6
+ class RateLimiter
7
+ # Checks if resource of specified size can be
8
+ # conforms with the current limit.
9
+ #
10
+ # Implementations of this method are not guaranteed
11
+ # to be side-effect free.
12
+ #
13
+ # @return [Boolean] whether a resource conforms with the current limit
14
+ def allow?(size); end
15
+
16
+ # The effective rate limiting ratio based on
17
+ # recent calls to `allow?`.
18
+ #
19
+ # @return [Float] recent allowance ratio
20
+ def effective_rate; end
21
+ end
22
+
23
+ # Implementation of the Token Bucket metering algorithm
24
+ # for rate limiting.
25
+ #
26
+ # @see https://en.wikipedia.org/wiki/Token_bucket Token bucket
27
+ class TokenBucket < RateLimiter
28
+ attr_reader :rate, :max_tokens
29
+
30
+ # @param rate [Numeric] Allowance rate, in units per second
31
+ # if rate is negative, always allow
32
+ # if rate is zero, never allow
33
+ # @param max_tokens [Numeric] Limit of available tokens
34
+ def initialize(rate, max_tokens = rate)
35
+ @rate = rate
36
+ @max_tokens = max_tokens
37
+
38
+ @tokens = max_tokens
39
+ @total_messages = 0
40
+ @conforming_messages = 0
41
+ @last_refill = Utils::Time.get_time
42
+ end
43
+
44
+ # Checks if a message of provided +size+
45
+ # conforms with the current bucket limit.
46
+ #
47
+ # If it does, return +true+ and remove +size+
48
+ # tokens from the bucket.
49
+ # If it does not, return +false+ without affecting
50
+ # the tokens form the bucket.
51
+ #
52
+ # @return [Boolean] +true+ if message conforms with current bucket limit
53
+ def allow?(size)
54
+ return false if @rate.zero?
55
+ return true if @rate < 0
56
+
57
+ refill_since_last_message
58
+
59
+ increment_total_count
60
+
61
+ return false if @tokens < size
62
+
63
+ increment_conforming_count
64
+
65
+ @tokens -= size
66
+
67
+ true
68
+ end
69
+
70
+ # Ratio of 'conformance' per 'total messages' checked
71
+ # on this bucket.
72
+ #
73
+ # Returns +1.0+ when no messages have been checked yet.
74
+ #
75
+ # @return [Float] Conformance ratio, between +[0,1]+
76
+ def effective_rate
77
+ return 0.0 if @rate.zero?
78
+ return 1.0 if @rate < 0 || @total_messages.zero?
79
+
80
+ @conforming_messages.to_f / @total_messages
81
+ end
82
+
83
+ # @return [Numeric] number of tokens currently available
84
+ def available_tokens
85
+ @tokens
86
+ end
87
+
88
+ private
89
+
90
+ def refill_since_last_message
91
+ now = Utils::Time.get_time
92
+ elapsed = now - @last_refill
93
+
94
+ refill_tokens(@rate * elapsed)
95
+
96
+ @last_refill = now
97
+ end
98
+
99
+ def refill_tokens(size)
100
+ @tokens += size
101
+ @tokens = @max_tokens if @tokens > @max_tokens
102
+ end
103
+
104
+ def increment_total_count
105
+ @total_messages += 1
106
+ end
107
+
108
+ def increment_conforming_count
109
+ @conforming_messages += 1
110
+ end
111
+ end
112
+
113
+ # \RateLimiter that accepts all resources,
114
+ # with no limits.
115
+ class UnlimitedLimiter < RateLimiter
116
+ # @return [Boolean] always +true+
117
+ def allow?(_)
118
+ true
119
+ end
120
+
121
+ # @return [Float] always 100%
122
+ def effective_rate
123
+ 1.0
124
+ end
125
+ end
126
+ end
127
+ end
@@ -0,0 +1,61 @@
1
+ require 'forwardable'
2
+
3
+ require 'ddtrace/sampling/matcher'
4
+ require 'ddtrace/sampler'
5
+
6
+ module Datadog
7
+ module Sampling
8
+ # Sampling rule that dictates if a span matches
9
+ # a specific criteria and what sampling strategy to
10
+ # apply in case of a positive match.
11
+ class Rule
12
+ extend Forwardable
13
+
14
+ attr_reader :matcher, :sampler
15
+
16
+ # @param [Matcher] matcher A matcher to verify span conformity against
17
+ # @param [Sampler] sampler A sampler to be consulted on a positive match
18
+ def initialize(matcher, sampler)
19
+ @matcher = matcher
20
+ @sampler = sampler
21
+ end
22
+
23
+ # Evaluates if the provided `span` conforms to the `matcher`.
24
+ #
25
+ # @param [Span] span
26
+ # @return [Boolean] whether this rules applies to the span
27
+ # @return [NilClass] if the matcher fails errs during evaluation
28
+ def match?(span)
29
+ @matcher.match?(span)
30
+ rescue => e
31
+ Datadog::Tracer.log.error("Matcher failed. Cause: #{e.message} Source: #{e.backtrace.first}")
32
+ nil
33
+ end
34
+
35
+ def_delegators :@sampler, :sample?, :sample_rate
36
+ end
37
+
38
+ # A \Rule that matches a span based on
39
+ # operation name and/or service name and
40
+ # applies a fixed sampling to matching spans.
41
+ class SimpleRule < Rule
42
+ # @param name [String,Regexp,Proc] Matcher for case equality (===) with the span name, defaults to always match
43
+ # @param service [String,Regexp,Proc] Matcher for case equality (===) with the service name, defaults to always match
44
+ # @param sample_rate [Float] Sampling rate between +[0,1]+
45
+ def initialize(name: SimpleMatcher::MATCH_ALL, service: SimpleMatcher::MATCH_ALL, sample_rate: 1.0)
46
+ # We want to allow 0.0 to drop all traces, but \RateSampler
47
+ # considers 0.0 an invalid rate and falls back to 100% sampling.
48
+ #
49
+ # We address that here by not setting the rate in the constructor,
50
+ # but using the setter method.
51
+ #
52
+ # We don't want to make this change directly to \RateSampler
53
+ # because it breaks its current contract to existing users.
54
+ sampler = Datadog::RateSampler.new
55
+ sampler.sample_rate = sample_rate
56
+
57
+ super(SimpleMatcher.new(name: name, service: service), sampler)
58
+ end
59
+ end
60
+ end
61
+ end