scout_apm 2.2.0.pre3 → 2.3.0.pre

Sign up to get free protection for your applications and to get access to all the features.
Files changed (127) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +1 -1
  3. data/CHANGELOG.markdown +147 -2
  4. data/Guardfile +43 -0
  5. data/Rakefile +2 -2
  6. data/ext/allocations/allocations.c +6 -0
  7. data/ext/allocations/extconf.rb +1 -0
  8. data/ext/rusage/README.md +26 -0
  9. data/ext/rusage/extconf.rb +5 -0
  10. data/ext/rusage/rusage.c +52 -0
  11. data/lib/scout_apm.rb +28 -15
  12. data/lib/scout_apm/agent.rb +89 -37
  13. data/lib/scout_apm/agent/logging.rb +6 -1
  14. data/lib/scout_apm/agent/reporting.rb +9 -6
  15. data/lib/scout_apm/app_server_load.rb +21 -10
  16. data/lib/scout_apm/attribute_arranger.rb +6 -3
  17. data/lib/scout_apm/background_job_integrations/delayed_job.rb +71 -1
  18. data/lib/scout_apm/background_job_integrations/resque.rb +85 -0
  19. data/lib/scout_apm/background_job_integrations/sidekiq.rb +22 -20
  20. data/lib/scout_apm/background_recorder.rb +43 -0
  21. data/lib/scout_apm/background_worker.rb +19 -15
  22. data/lib/scout_apm/config.rb +138 -28
  23. data/lib/scout_apm/db_query_metric_set.rb +80 -0
  24. data/lib/scout_apm/db_query_metric_stats.rb +102 -0
  25. data/lib/scout_apm/debug.rb +37 -0
  26. data/lib/scout_apm/environment.rb +22 -15
  27. data/lib/scout_apm/git_revision.rb +51 -0
  28. data/lib/scout_apm/histogram.rb +11 -2
  29. data/lib/scout_apm/instant/assets/xmlhttp_instrumentation.html +2 -2
  30. data/lib/scout_apm/instant/middleware.rb +196 -54
  31. data/lib/scout_apm/instruments/action_controller_rails_3_rails4.rb +89 -68
  32. data/lib/scout_apm/instruments/action_view.rb +49 -0
  33. data/lib/scout_apm/instruments/active_record.rb +127 -3
  34. data/lib/scout_apm/instruments/grape.rb +4 -3
  35. data/lib/scout_apm/instruments/middleware_detailed.rb +4 -6
  36. data/lib/scout_apm/instruments/mongoid.rb +24 -3
  37. data/lib/scout_apm/instruments/net_http.rb +7 -2
  38. data/lib/scout_apm/instruments/percentile_sampler.rb +36 -19
  39. data/lib/scout_apm/instruments/process/process_cpu.rb +3 -2
  40. data/lib/scout_apm/instruments/process/process_memory.rb +3 -3
  41. data/lib/scout_apm/instruments/resque.rb +40 -0
  42. data/lib/scout_apm/layaway.rb +67 -28
  43. data/lib/scout_apm/layer.rb +19 -59
  44. data/lib/scout_apm/layer_children_set.rb +77 -0
  45. data/lib/scout_apm/layer_converters/allocation_metric_converter.rb +5 -6
  46. data/lib/scout_apm/layer_converters/converter_base.rb +201 -14
  47. data/lib/scout_apm/layer_converters/database_converter.rb +55 -0
  48. data/lib/scout_apm/layer_converters/depth_first_walker.rb +22 -10
  49. data/lib/scout_apm/layer_converters/error_converter.rb +5 -7
  50. data/lib/scout_apm/layer_converters/find_layer_by_type.rb +34 -0
  51. data/lib/scout_apm/layer_converters/histograms.rb +14 -0
  52. data/lib/scout_apm/layer_converters/job_converter.rb +36 -50
  53. data/lib/scout_apm/layer_converters/metric_converter.rb +17 -19
  54. data/lib/scout_apm/layer_converters/request_queue_time_converter.rb +10 -12
  55. data/lib/scout_apm/layer_converters/slow_job_converter.rb +41 -115
  56. data/lib/scout_apm/layer_converters/slow_request_converter.rb +33 -117
  57. data/lib/scout_apm/limited_layer.rb +126 -0
  58. data/lib/scout_apm/metric_meta.rb +0 -5
  59. data/lib/scout_apm/metric_set.rb +9 -1
  60. data/lib/scout_apm/metric_stats.rb +7 -8
  61. data/lib/scout_apm/rack.rb +26 -0
  62. data/lib/scout_apm/remote/message.rb +23 -0
  63. data/lib/scout_apm/remote/recorder.rb +57 -0
  64. data/lib/scout_apm/remote/router.rb +49 -0
  65. data/lib/scout_apm/remote/server.rb +58 -0
  66. data/lib/scout_apm/reporter.rb +51 -15
  67. data/lib/scout_apm/request_histograms.rb +4 -0
  68. data/lib/scout_apm/request_manager.rb +2 -1
  69. data/lib/scout_apm/scored_item_set.rb +7 -0
  70. data/lib/scout_apm/serializers/db_query_serializer_to_json.rb +15 -0
  71. data/lib/scout_apm/serializers/histograms_serializer_to_json.rb +21 -0
  72. data/lib/scout_apm/serializers/payload_serializer.rb +10 -3
  73. data/lib/scout_apm/serializers/payload_serializer_to_json.rb +6 -6
  74. data/lib/scout_apm/serializers/slow_jobs_serializer_to_json.rb +2 -1
  75. data/lib/scout_apm/server_integrations/puma.rb +5 -2
  76. data/lib/scout_apm/slow_job_policy.rb +1 -10
  77. data/lib/scout_apm/slow_job_record.rb +6 -1
  78. data/lib/scout_apm/slow_request_policy.rb +1 -10
  79. data/lib/scout_apm/slow_transaction.rb +20 -2
  80. data/lib/scout_apm/store.rb +66 -12
  81. data/lib/scout_apm/synchronous_recorder.rb +26 -0
  82. data/lib/scout_apm/tracked_request.rb +136 -71
  83. data/lib/scout_apm/utils/active_record_metric_name.rb +8 -4
  84. data/lib/scout_apm/utils/backtrace_parser.rb +3 -3
  85. data/lib/scout_apm/utils/gzip_helper.rb +24 -0
  86. data/lib/scout_apm/utils/numbers.rb +14 -0
  87. data/lib/scout_apm/utils/scm.rb +14 -0
  88. data/lib/scout_apm/version.rb +1 -1
  89. data/scout_apm.gemspec +5 -4
  90. data/test/test_helper.rb +18 -0
  91. data/test/unit/config_test.rb +59 -8
  92. data/test/unit/db_query_metric_set_test.rb +56 -0
  93. data/test/unit/db_query_metric_stats_test.rb +113 -0
  94. data/test/unit/git_revision_test.rb +15 -0
  95. data/test/unit/histogram_test.rb +14 -0
  96. data/test/unit/instruments/net_http_test.rb +21 -0
  97. data/test/unit/instruments/percentile_sampler_test.rb +137 -0
  98. data/test/unit/layaway_test.rb +20 -0
  99. data/test/unit/layer_children_set_test.rb +88 -0
  100. data/test/unit/layer_converters/depth_first_walker_test.rb +66 -0
  101. data/test/unit/layer_converters/metric_converter_test.rb +22 -0
  102. data/test/unit/layer_converters/stubs.rb +33 -0
  103. data/test/unit/limited_layer_test.rb +53 -0
  104. data/test/unit/remote/test_message.rb +13 -0
  105. data/test/unit/remote/test_router.rb +33 -0
  106. data/test/unit/remote/test_server.rb +15 -0
  107. data/test/unit/serializers/payload_serializer_test.rb +3 -12
  108. data/test/unit/store_test.rb +66 -0
  109. data/test/unit/test_tracked_request.rb +87 -0
  110. data/test/unit/utils/active_record_metric_name_test.rb +8 -0
  111. data/test/unit/utils/backtrace_parser_test.rb +5 -0
  112. data/test/unit/utils/numbers_test.rb +15 -0
  113. data/test/unit/utils/scm.rb +17 -0
  114. metadata +125 -30
  115. data/ext/stacks/extconf.rb +0 -37
  116. data/ext/stacks/scout_atomics.h +0 -86
  117. data/ext/stacks/stacks.c +0 -811
  118. data/lib/scout_apm/capacity.rb +0 -57
  119. data/lib/scout_apm/deploy_integrations/capistrano_2.cap +0 -12
  120. data/lib/scout_apm/deploy_integrations/capistrano_2.rb +0 -83
  121. data/lib/scout_apm/deploy_integrations/capistrano_3.cap +0 -12
  122. data/lib/scout_apm/deploy_integrations/capistrano_3.rb +0 -88
  123. data/lib/scout_apm/instruments/delayed_job.rb +0 -57
  124. data/lib/scout_apm/serializers/deploy_serializer.rb +0 -16
  125. data/lib/scout_apm/trace_compactor.rb +0 -312
  126. data/lib/scout_apm/utils/fake_stacks.rb +0 -87
  127. data/tester.rb +0 -53
@@ -0,0 +1,21 @@
1
+ require 'test_helper'
2
+
3
+ require 'scout_apm/instruments/net_http'
4
+
5
+ require 'addressable'
6
+
7
+ class NetHttpTest < Minitest::Test
8
+ def setup
9
+ ScoutApm::Instruments::NetHttp.new.install
10
+ end
11
+
12
+ def test_request_scout_description_for_uri
13
+ req = Net::HTTP::Get.new(URI('http://example.org/here'))
14
+ assert_equal '/here', Net::HTTP.new('').request_scout_description(req)
15
+ end
16
+
17
+ def test_request_scout_description_for_addressable
18
+ req = Net::HTTP::Get.new(Addressable::URI.parse('http://example.org/here'))
19
+ assert_equal '/here', Net::HTTP.new('').request_scout_description(req)
20
+ end
21
+ end
@@ -0,0 +1,137 @@
1
+ require 'test_helper'
2
+
3
+ require 'scout_apm/instruments/percentile_sampler'
4
+
5
+ class PercentileSamplerTest < Minitest::Test
6
+ PercentileSampler = ScoutApm::Instruments::PercentileSampler
7
+ HistogramReport = ScoutApm::Instruments::HistogramReport
8
+
9
+ attr_reader :subject
10
+
11
+ def setup
12
+ @subject = PercentileSampler.new(logger, histograms)
13
+ end
14
+
15
+
16
+ def test_initialize_with_logger_and_histogram_set
17
+ assert_equal subject.logger, logger
18
+ assert_equal subject.histograms, histograms
19
+ end
20
+
21
+ def test_implements_instrument_interface
22
+ assert subject.respond_to?(:human_name)
23
+ end
24
+
25
+ def test_percentiles_returns_one_percentile_per_endpoint_at_time
26
+ histograms[time].add("foo", 10)
27
+ histograms[time].add("bar", 15)
28
+ histograms[time2].add("baz", 15)
29
+
30
+ assert_equal subject.percentiles(time).length, 2
31
+ assert_equal subject.percentiles(time2).length, 1
32
+ end
33
+
34
+ def test_percentiles_clears_time_from_hash
35
+ histograms[time].add("foo", 10)
36
+ histograms[time2].add("baz", 15)
37
+
38
+ subject.percentiles(time)
39
+
40
+ assert_false histograms.key?(time)
41
+ assert histograms.key?(time + 10)
42
+ end
43
+
44
+ def test_percentiles_returns_histogram_reports
45
+ histograms[time].add("foo", 10)
46
+
47
+ assert subject.percentiles(time).
48
+ all?{ |item| item.is_a?(HistogramReport) }
49
+ end
50
+
51
+ def test_percentiles_returns_correct_histogram_report
52
+ histograms[time].add("foo", 100)
53
+ histograms[time].add("foo", 200)
54
+ histograms[time].add("foo", 100)
55
+ histograms[time].add("foo", 300)
56
+
57
+ report = subject.percentiles(time).first
58
+ histogram = report.histogram
59
+
60
+ assert_equal "foo", report.name
61
+ assert_equal 4, histogram.total
62
+ assert_equal [[2, 100], [1, 200], [1, 300]],
63
+ histogram.bins.map{|bin| [bin.count, bin.value] }
64
+ end
65
+
66
+ def test_metrics_saves_histogram_to_store
67
+ store = mock
68
+ store.expects(:track_histograms!)
69
+ subject.metrics(ScoutApm::StoreReportingPeriodTimestamp.new(time), store)
70
+ end
71
+
72
+
73
+ ################################################################################
74
+ # HistogramReport Test
75
+ ################################################################################
76
+ def test_histogram_report_combine_refuses_to_combine_mismatched_name
77
+ assert_raises { HistogramReport.new("foo", histogram).combine!(HistogramReport.new("bar", histogram)) }
78
+ end
79
+
80
+ def test_histogram_report_merge_keeps_name
81
+ report1 = HistogramReport.new("foo", histogram)
82
+ report2 = HistogramReport.new("foo", histogram)
83
+ combined = report1.combine!(report2)
84
+
85
+ assert "foo", combined.name
86
+ end
87
+
88
+ def test_histogram_report_combine_merges_histograms
89
+ histogram1 = histogram
90
+ histogram2 = histogram
91
+ histogram1.add(1)
92
+ histogram1.add(2)
93
+ histogram2.add(2)
94
+ histogram2.add(3)
95
+
96
+ report1 = HistogramReport.new("foo", histogram1)
97
+ report2 = HistogramReport.new("foo", histogram2)
98
+ combined = report1.combine!(report2)
99
+
100
+ assert_equal 4, combined.histogram.total
101
+ assert_equal [[1, 1], [2, 2], [1, 3]],
102
+ combined.histogram.bins.map{|bin| [bin.count, bin.value.to_i] }
103
+ end
104
+
105
+ ################################################################################
106
+ # Test Helpers
107
+ ################################################################################
108
+ def logger
109
+ @logger ||= begin
110
+ @logger_io = StringIO.new
111
+ Logger.new(@logger_io)
112
+ end
113
+ end
114
+
115
+ def histograms
116
+ @histograms ||= begin
117
+ @request_histograms_by_time = Hash.new { |hash, key|
118
+ hash[key] = ScoutApm::RequestHistograms.new
119
+ }
120
+ end
121
+ end
122
+
123
+ def histogram
124
+ max_bins = 20
125
+ ScoutApm::NumericHistogram.new(max_bins)
126
+ end
127
+
128
+ # An arbitrary time
129
+ def time
130
+ @time ||= Time.now
131
+ end
132
+
133
+ def time2
134
+ time + 10
135
+ end
136
+ end
137
+
@@ -21,4 +21,24 @@ class LayawayTest < Minitest::Test
21
21
 
22
22
  assert_equal Pathname.new("/tmp/scout_apm_test/tmp"), ScoutApm::Layaway.new(config, env).directory
23
23
  end
24
+
25
+ def test_layaway_file_limit_prevents_new_writes
26
+ FileUtils.mkdir_p '/tmp/scout_apm_test/layaway_limit'
27
+ config = make_fake_config("data_file" => "/tmp/scout_apm_test/layaway_limit")
28
+ layaway = ScoutApm::Layaway.new(config, ScoutApm::Agent.instance.environment)
29
+ layaway.delete_files_for(:all)
30
+
31
+ current_time = Time.now.utc
32
+ current_rp = ScoutApm::StoreReportingPeriod.new(current_time)
33
+ stale_rp = ScoutApm::StoreReportingPeriod.new(current_time - current_time.sec - 120)
34
+
35
+ # layaway.write_reporting_period returns nil on successful write
36
+ # It should probably be changed to return true or the number of bytes written
37
+ assert_nil layaway.write_reporting_period(stale_rp, 1)
38
+
39
+ # layaway.write_reporting_period returns an explicit false class on failure
40
+ assert layaway.write_reporting_period(current_rp, 1).is_a?(FalseClass)
41
+
42
+ layaway.delete_files_for(:all)
43
+ end
24
44
  end
@@ -0,0 +1,88 @@
1
+ require 'test_helper'
2
+ require 'scout_apm/layer_children_set'
3
+
4
+ class LayerChildrenSetTest < Minitest::Test
5
+ SET = ScoutApm::LayerChildrenSet
6
+
7
+ def test_limit_default
8
+ assert_equal SET::DEFAULT_UNIQUE_CUTOFF, SET.new.unique_cutoff
9
+ end
10
+
11
+ # Add 5, make sure they're all in the children list we get back.
12
+ def test_add_layer_before_limit
13
+ s = SET.new(5)
14
+
15
+ 5.times do
16
+ s << make_layer("LayerType", "LayerName")
17
+ end
18
+
19
+ children = s.to_a
20
+ assert_equal 5, children.size
21
+
22
+ # Don't care about order
23
+ (0..4).each do |i|
24
+ assert children.include?(lookup_layer(i))
25
+ end
26
+ end
27
+
28
+ def test_add_layer_after_limit
29
+ s = SET.new(5)
30
+
31
+ 10.times do
32
+ s << make_layer("LayerType", "LayerName")
33
+ end
34
+
35
+ children = s.to_a
36
+ # 6 = 5 real ones + 1 merged.
37
+ assert_equal 6, children.size
38
+
39
+ # Don't care about order
40
+ (0..4).each do |i|
41
+ assert children.include?(lookup_layer(i))
42
+ end
43
+
44
+ # Don't care about order
45
+ (5..9).each do |i|
46
+ assert ! children.include?(lookup_layer(i))
47
+ end
48
+
49
+ limited_layer = children.last
50
+ assert_equal ScoutApm::LimitedLayer, limited_layer.class
51
+ assert_equal 5, limited_layer.count
52
+ end
53
+
54
+ def test_add_layer_with_different_type_after_limit
55
+ s = SET.new(5)
56
+
57
+ # Add 20 items
58
+ 10.times do
59
+ s << make_layer("LayerType", "LayerName")
60
+ s << make_layer("DifferentLayerType", "LayerName")
61
+ end
62
+
63
+ children = s.to_a
64
+
65
+ # Tyo types, so 2 distinct limitdlayer objects
66
+ limited_layers = children.select{ |l| ScoutApm::LimitedLayer === l }
67
+ assert_equal 2, limited_layers.length
68
+
69
+ # 5 unchanged children each for the two layer types, plus the 2 limitd when each overran their limit
70
+ assert_equal 12, children.length
71
+ limited_layers.each { |ml| assert_equal 5, ml.count }
72
+ end
73
+
74
+ #############
75
+ # Helpers #
76
+ #############
77
+
78
+ def make_layer(type, name)
79
+ @made_layers ||= []
80
+ l = ScoutApm::Layer.new(type, name)
81
+ @made_layers << l
82
+ l
83
+ end
84
+
85
+ def lookup_layer(i)
86
+ @made_layers[i]
87
+ end
88
+ end
@@ -0,0 +1,66 @@
1
+ require 'test_helper'
2
+
3
+ module ScoutApm
4
+ class DepthFirstWalkerTest < Minitest::Test
5
+ def test_walk_single_node_calls_callbacks_in_order
6
+ calls = []
7
+ layer = Layer.new("A", "x")
8
+
9
+ walker = LayerConverters::DepthFirstWalker.new(layer)
10
+ walker.before { |l| calls << :before }
11
+ walker.after { |l| calls << :after }
12
+ walker.on { |l| calls << :on }
13
+
14
+ walker.walk
15
+
16
+ assert_equal [:before, :on, :after], calls
17
+ end
18
+
19
+ # Tree looks like:
20
+ # A
21
+ # |
22
+ # B
23
+ # / \ \
24
+ # C D E
25
+ # / \
26
+ # F G
27
+ def test_walk_interesting_tree
28
+ calls = []
29
+ a = Layer.new("A", "x")
30
+ b = Layer.new("B", "x")
31
+ c = Layer.new("C", "x")
32
+ d = Layer.new("D", "x")
33
+ e = Layer.new("E", "x")
34
+ f = Layer.new("F", "x")
35
+ g = Layer.new("G", "x")
36
+ a.add_child(b)
37
+ b.add_child(c)
38
+ b.add_child(d)
39
+ b.add_child(e)
40
+ c.add_child(f)
41
+ c.add_child(g)
42
+
43
+ root_layer = a
44
+
45
+ walker = LayerConverters::DepthFirstWalker.new(root_layer)
46
+ walker.before { |l| calls << "#{l.type} before" }
47
+ walker.after { |l| calls << "#{l.type} after" }
48
+ walker.on { |l| calls << "#{l.type} on" }
49
+
50
+ walker.walk
51
+
52
+ assert_equal [
53
+ "A before", "A on", # before & on always line up next to each other
54
+ "B before", "B on",
55
+ "C before", "C on",
56
+ "F before", "F on", "F after", # leaf nodes run all 3
57
+ "G before", "G on", "G after",
58
+ "C after", # once all children are done, do a node's after
59
+ "D before", "D on", "D after",
60
+ "E before", "E on", "E after",
61
+ "B after",
62
+ "A after"
63
+ ], calls
64
+ end
65
+ end
66
+ end
@@ -0,0 +1,22 @@
1
+ require 'test_helper'
2
+ require_relative 'stubs'
3
+
4
+ module ScoutApm
5
+ module LayerConverters
6
+ class MetricConverterTest < Minitest::Test
7
+ include Stubs
8
+
9
+ def test_register_adds_hooks
10
+ mc = MetricConverter.new(faux_request, faux_layer_finder, faux_store)
11
+ faux_walker.expects(:on)
12
+ mc.register_hooks(faux_walker)
13
+ end
14
+
15
+ def test_record
16
+ mc = MetricConverter.new(faux_request, faux_layer_finder, faux_store)
17
+ faux_store.expects(:track!)
18
+ mc.record!
19
+ end
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,33 @@
1
+ ####################################
2
+ # Stubs for LayerConverter Tests #
3
+ ####################################
4
+ module ScoutApm
5
+ module LayerConverters
6
+ module Stubs
7
+ def faux_walker(subscope_stubs: true)
8
+ @w ||= stub
9
+
10
+ if subscope_stubs && !@w_set_subscope_stubs
11
+ @w_set_subscope_stubs = true
12
+ @w.expects(:before)
13
+ @w.expects(:after)
14
+ end
15
+ @w
16
+ end
17
+
18
+ def faux_request
19
+ @req ||= stub(:root_layer => stub)
20
+ end
21
+
22
+ def faux_layer_finder
23
+ @layer_finder ||= stub
24
+ @layer_finder.stubs(scope: stub)
25
+ @layer_finder
26
+ end
27
+
28
+ def faux_store
29
+ @store ||= stub
30
+ end
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,53 @@
1
+ require 'test_helper'
2
+ require 'ostruct'
3
+
4
+ class LimitedLayerTest < Minitest::Test
5
+
6
+ def test_counts_while_absorbing
7
+ ll = ScoutApm::LimitedLayer.new("ActiveRecord")
8
+ assert_equal 0, ll.count
9
+
10
+ ll.absorb faux_layer("ActiveRecord", "User#Find", 2, 1, 200, 100)
11
+ assert_equal 1, ll.count
12
+
13
+ ll.absorb faux_layer("ActiveRecord", "User#Find", 2, 1, 200, 100)
14
+ assert_equal 2, ll.count
15
+ end
16
+
17
+ def test_sums_values_while_absorbing
18
+ ll = ScoutApm::LimitedLayer.new("ActiveRecord")
19
+
20
+ ll.absorb faux_layer("ActiveRecord", "User#Find", 2, 1, 200, 100)
21
+ assert_equal 1, ll.total_exclusive_time
22
+ assert_equal 2, ll.total_call_time
23
+ assert_equal 100, ll.total_exclusive_allocations
24
+ assert_equal 200, ll.total_allocations
25
+
26
+
27
+ ll.absorb faux_layer("ActiveRecord", "User#Find", 4, 3, 400, 300)
28
+ assert_equal 4, ll.total_exclusive_time # 3 + 1
29
+ assert_equal 6, ll.total_call_time # 4 + 2
30
+ assert_equal 400, ll.total_exclusive_allocations # 300 + 100
31
+ assert_equal 600, ll.total_allocations # 400 + 200
32
+ end
33
+
34
+ def test_the_name
35
+ ll = ScoutApm::LimitedLayer.new("ActiveRecord")
36
+ assert_equal "ActiveRecord/Limited", ll.legacy_metric_name
37
+ end
38
+
39
+ #############
40
+ # Helpers #
41
+ #############
42
+
43
+ def faux_layer(type, name, tct, tet, a_tct, a_tet)
44
+ OpenStruct.new(
45
+ :type => type,
46
+ :name => name,
47
+ :total_call_time => tct,
48
+ :total_exclusive_time => tet,
49
+ :total_allocations => a_tct,
50
+ :total_exclusive_allocations => a_tet,
51
+ )
52
+ end
53
+ end