scout_apm 3.0.0.pre11 → 3.0.0.pre12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.markdown +13 -4
  3. data/Guardfile +1 -0
  4. data/lib/scout_apm.rb +6 -0
  5. data/lib/scout_apm/agent/reporting.rb +2 -1
  6. data/lib/scout_apm/attribute_arranger.rb +14 -1
  7. data/lib/scout_apm/config.rb +12 -0
  8. data/lib/scout_apm/db_query_metric_set.rb +80 -0
  9. data/lib/scout_apm/db_query_metric_stats.rb +102 -0
  10. data/lib/scout_apm/fake_store.rb +6 -0
  11. data/lib/scout_apm/instant/middleware.rb +6 -1
  12. data/lib/scout_apm/instruments/active_record.rb +111 -0
  13. data/lib/scout_apm/layer.rb +4 -0
  14. data/lib/scout_apm/layer_converters/allocation_metric_converter.rb +5 -6
  15. data/lib/scout_apm/layer_converters/converter_base.rb +7 -19
  16. data/lib/scout_apm/layer_converters/database_converter.rb +81 -0
  17. data/lib/scout_apm/layer_converters/depth_first_walker.rb +22 -10
  18. data/lib/scout_apm/layer_converters/error_converter.rb +5 -7
  19. data/lib/scout_apm/layer_converters/find_layer_by_type.rb +34 -0
  20. data/lib/scout_apm/layer_converters/histograms.rb +14 -0
  21. data/lib/scout_apm/layer_converters/job_converter.rb +35 -49
  22. data/lib/scout_apm/layer_converters/metric_converter.rb +16 -18
  23. data/lib/scout_apm/layer_converters/request_queue_time_converter.rb +10 -12
  24. data/lib/scout_apm/layer_converters/slow_job_converter.rb +33 -35
  25. data/lib/scout_apm/layer_converters/slow_request_converter.rb +22 -18
  26. data/lib/scout_apm/limited_layer.rb +4 -0
  27. data/lib/scout_apm/metric_meta.rb +0 -5
  28. data/lib/scout_apm/metric_stats.rb +8 -1
  29. data/lib/scout_apm/serializers/db_query_serializer_to_json.rb +15 -0
  30. data/lib/scout_apm/serializers/payload_serializer.rb +4 -3
  31. data/lib/scout_apm/serializers/payload_serializer_to_json.rb +5 -2
  32. data/lib/scout_apm/slow_job_policy.rb +1 -10
  33. data/lib/scout_apm/slow_request_policy.rb +1 -10
  34. data/lib/scout_apm/store.rb +41 -22
  35. data/lib/scout_apm/tracked_request.rb +28 -40
  36. data/lib/scout_apm/utils/active_record_metric_name.rb +8 -4
  37. data/lib/scout_apm/version.rb +1 -1
  38. data/test/unit/db_query_metric_set_test.rb +56 -0
  39. data/test/unit/db_query_metric_stats_test.rb +113 -0
  40. data/test/unit/fake_store_test.rb +10 -0
  41. data/test/unit/layer_converters/depth_first_walker_test.rb +66 -0
  42. data/test/unit/layer_converters/metric_converter_test.rb +22 -0
  43. data/test/unit/layer_converters/stubs.rb +33 -0
  44. data/test/unit/serializers/payload_serializer_test.rb +3 -12
  45. data/test/unit/store_test.rb +4 -4
  46. data/test/unit/utils/active_record_metric_name_test.rb +8 -0
  47. metadata +20 -2
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 36bac9cf47e89f5cec6475b574b6cd948339d627
4
- data.tar.gz: dd2bc23001811e393950d502f04b2a359dae083d
3
+ metadata.gz: ca4bf16454ad9fca2937243b738f7b8a6ae6c080
4
+ data.tar.gz: 9c84f2ee03e001220109dc1ffb32f2d5f7ab8e50
5
5
  SHA512:
6
- metadata.gz: 0d2e5b8270afe767d63890df65b8ea6f1fba96f4a2187cd58537372e13c7555f94875b3884473f210eb6e96f1815240508a53c50638e0ba74ad96075d9364888
7
- data.tar.gz: 7b0169e3e311061dc20da5088f7b22247ec3acf66ba145b827a926ef87763bf6e576282a282611c2d5178412de7b8ade5a24a7c50e730399c560e1f846626020
6
+ metadata.gz: f8b62b98284c93d8190408a9d364f9f3a82894670c52241a3de2faadd08635c332bdb93c50c37c331cd1458ebee8b67b68e979db03a2ea375d4775ce6103ea03
7
+ data.tar.gz: 99500b8f0e6e536e868c2f0e337775e61af71c7da1ddad986e2213807493e96d74b56330c0e84f4aca78cb35f6ecb1ddbeed0a6b4bd5e20cc5e367387e58fc29
data/CHANGELOG.markdown CHANGED
@@ -1,13 +1,23 @@
1
- <<<<<<< HEAD
2
1
  # 3.0.0
3
2
 
4
3
  * ScoutProf BETA
5
- =======
4
+
5
+ # 2.3.0
6
+
7
+ Note: ScoutApm Agent version 2.2.0 was the initial ScoutProf agent that was
8
+ determined quickly to be a big enough change to warrant the move to 3.0. We are not
9
+ reusing that version number to avoid confusion.
10
+
11
+ * Deeper database query instrumentation. The agent now collects app-wide
12
+ database usage on every call. This will allow you to better identify
13
+ persistently slow queries, and capacity bottlenecks.
14
+ * Optimize the approach used during recording each request to avoid unnecessary
15
+ work, improving performance
16
+
6
17
  # 2.1.32
7
18
 
8
19
  * Better naming when using Resque + ActiveJob
9
20
  * Better naming when using Sidekiq + DelayedExtension
10
- >>>>>>> master
11
21
 
12
22
  # 2.1.31
13
23
 
@@ -46,7 +56,6 @@
46
56
  # 2.1.23
47
57
 
48
58
  * Extend Mongoid instrumentation to 6.x
49
- >>>>>>> master
50
59
 
51
60
  # 2.1.22
52
61
 
data/Guardfile CHANGED
@@ -18,6 +18,7 @@
18
18
  guard :minitest do
19
19
  # with Minitest::Unit
20
20
  watch(%r{^test/(.*)\/?test_(.*)\.rb$})
21
+ watch(%r{^test/(.*)\/?(.*)_test\.rb$})
21
22
  watch(%r{^lib/(.*/)?([^/]+)\.rb$}) { |m| "test/#{m[1]}test_#{m[2]}.rb" }
22
23
  watch(%r{^test/test_helper\.rb$}) { 'test' }
23
24
 
data/lib/scout_apm.rb CHANGED
@@ -41,9 +41,12 @@ require 'scout_apm/layer_converters/error_converter'
41
41
  require 'scout_apm/layer_converters/job_converter'
42
42
  require 'scout_apm/layer_converters/slow_job_converter'
43
43
  require 'scout_apm/layer_converters/metric_converter'
44
+ require 'scout_apm/layer_converters/database_converter'
44
45
  require 'scout_apm/layer_converters/slow_request_converter'
45
46
  require 'scout_apm/layer_converters/request_queue_time_converter'
46
47
  require 'scout_apm/layer_converters/allocation_metric_converter'
48
+ require 'scout_apm/layer_converters/histograms'
49
+ require 'scout_apm/layer_converters/find_layer_by_type'
47
50
 
48
51
  require 'scout_apm/server_integrations/passenger'
49
52
  require 'scout_apm/server_integrations/puma'
@@ -122,6 +125,7 @@ require 'scout_apm/background_worker'
122
125
  require 'scout_apm/bucket_name_splitter'
123
126
  require 'scout_apm/stack_item'
124
127
  require 'scout_apm/metric_set'
128
+ require 'scout_apm/db_query_metric_set'
125
129
  require 'scout_apm/store'
126
130
  require 'scout_apm/fake_store'
127
131
  require 'scout_apm/tracer'
@@ -133,6 +137,7 @@ require 'scout_apm/synchronous_recorder'
133
137
 
134
138
  require 'scout_apm/metric_meta'
135
139
  require 'scout_apm/metric_stats'
140
+ require 'scout_apm/db_query_metric_stats'
136
141
  require 'scout_apm/slow_transaction'
137
142
  require 'scout_apm/slow_job_record'
138
143
  require 'scout_apm/scored_item_set'
@@ -150,6 +155,7 @@ require 'scout_apm/serializers/jobs_serializer_to_json'
150
155
  require 'scout_apm/serializers/slow_jobs_serializer_to_json'
151
156
  require 'scout_apm/serializers/metrics_to_json_serializer'
152
157
  require 'scout_apm/serializers/histograms_serializer_to_json'
158
+ require 'scout_apm/serializers/db_query_serializer_to_json'
153
159
  require 'scout_apm/serializers/directive_serializer'
154
160
  require 'scout_apm/serializers/app_server_load_serializer'
155
161
 
@@ -59,6 +59,7 @@ module ScoutApm
59
59
  jobs = reporting_period.jobs
60
60
  slow_jobs = reporting_period.slow_jobs_payload
61
61
  histograms = reporting_period.histograms
62
+ db_query_metrics = reporting_period.db_query_metrics_payload
62
63
 
63
64
  metadata = {
64
65
  :app_root => ScoutApm::Environment.instance.root.to_s,
@@ -71,7 +72,7 @@ module ScoutApm
71
72
 
72
73
  log_deliver(metrics, slow_transactions, metadata, slow_jobs, histograms)
73
74
 
74
- payload = ScoutApm::Serializers::PayloadSerializer.serialize(metadata, metrics, slow_transactions, jobs, slow_jobs, histograms)
75
+ payload = ScoutApm::Serializers::PayloadSerializer.serialize(metadata, metrics, slow_transactions, jobs, slow_jobs, histograms, db_query_metrics)
75
76
  logger.debug("Sending payload w/ Headers: #{headers.inspect}")
76
77
 
77
78
  reporter.report(payload, headers)
@@ -14,7 +14,20 @@ module ScoutApm
14
14
  when :name
15
15
  attribute_hash[attribute] = subject.bucket_name
16
16
  when Symbol
17
- attribute_hash[attribute] = subject.send(attribute)
17
+ data = subject.send(attribute)
18
+
19
+ # Never try to `as_json` a time object, since it'll break if the
20
+ # app has the Oj gem set to mimic_JSON, and there's never
21
+ # anything interesting nested inside of a Time obj. We just want
22
+ # the ISO8601 string (which happens later in the payload
23
+ # serializing process)
24
+ if data.is_a?(Time)
25
+ attribute_hash[attribute] = data
26
+ elsif data.respond_to?(:as_json)
27
+ attribute_hash[attribute] = data.as_json
28
+ else
29
+ attribute_hash[attribute] = data
30
+ end
18
31
  end
19
32
  attribute_hash
20
33
  end
@@ -41,6 +41,8 @@ module ScoutApm
41
41
  'compress_payload',
42
42
  'config_file',
43
43
  'data_file',
44
+ 'database_metric_limit',
45
+ 'database_metric_report_limit',
44
46
  'detailed_middleware',
45
47
  'dev_trace',
46
48
  'direct_host',
@@ -125,6 +127,12 @@ module ScoutApm
125
127
  end
126
128
  end
127
129
 
130
+ class IntegerCoercion
131
+ def coerce(val)
132
+ val.to_i
133
+ end
134
+ end
135
+
128
136
  # Simply returns the passed in value, without change
129
137
  class NullCoercion
130
138
  def coerce(val)
@@ -140,6 +148,8 @@ module ScoutApm
140
148
  "enable_background_jobs" => BooleanCoercion.new,
141
149
  "ignore" => JsonCoercion.new,
142
150
  "monitor" => BooleanCoercion.new,
151
+ 'database_metric_limit' => IntegerCoercion.new,
152
+ 'database_metric_report_limit' => IntegerCoercion.new,
143
153
  }
144
154
 
145
155
 
@@ -227,6 +237,8 @@ module ScoutApm
227
237
  'uri_reporting' => 'full_path',
228
238
  'remote_agent_host' => '127.0.0.1',
229
239
  'remote_agent_port' => 7721, # picked at random
240
+ 'database_metric_limit' => 5000, # The hard limit on db metrics
241
+ 'database_metric_report_limit' => 1000,
230
242
  }.freeze
231
243
 
232
244
  def value(key)
@@ -0,0 +1,80 @@
1
+ module ScoutApm
2
+ class DbQueryMetricSet
3
+ include Enumerable
4
+
5
+ attr_reader :metrics # the raw metrics. You probably want #metrics_to_report
6
+ attr_reader :config # A ScoutApm::Config instance
7
+
8
+ def initialize(config=ScoutApm::Agent.instance.config)
9
+ # A hash of DbQueryMetricStats values, keyed by DbQueryMetricStats.key
10
+ @metrics = Hash.new
11
+ @config = config
12
+ end
13
+
14
+ def each
15
+ metrics.each do |_key, db_query_metric_stat|
16
+ yield db_query_metric_stat
17
+ end
18
+ end
19
+
20
+ # Looks up a DbQueryMetricStats instance in the +@metrics+ hash. Sets the value to +other+ if no key
21
+ # Returns a DbQueryMetricStats instance
22
+ def lookup(other)
23
+ metrics[other.key] ||= other
24
+ end
25
+
26
+ # Take another set, and merge it with this one
27
+ def combine!(other)
28
+ other.each do |metric|
29
+ self << metric
30
+ end
31
+ self
32
+ end
33
+
34
+ # Add a single DbQueryMetricStats object to this set.
35
+ #
36
+ # Looks up an existing one under this key and merges, or just saves a new
37
+ # one under the key
38
+ def <<(stat)
39
+ existing_stat = metrics[stat.key]
40
+ if existing_stat
41
+ existing_stat.combine!(stat)
42
+ elsif at_limit?
43
+ # We're full up, can't add any more.
44
+ # Should I log this? It may get super noisy?
45
+ else
46
+ metrics[stat.key] = stat
47
+ end
48
+ end
49
+
50
+ def increment_transaction_count!
51
+ metrics.each do |_key, db_query_metric_stat|
52
+ db_query_metric_stat.increment_transaction_count!
53
+ end
54
+ end
55
+
56
+ def metrics_to_report
57
+ report_limit = config.value('database_metric_report_limit')
58
+ if metrics.size > report_limit
59
+ metrics.
60
+ values.
61
+ sort_by {|stat| stat.call_time }.
62
+ reverse.
63
+ take(report_limit)
64
+ else
65
+ metrics.values
66
+ end
67
+ end
68
+
69
+ def inspect
70
+ metrics.map {|key, metric|
71
+ "#{key.inspect} - Count: #{metric.call_count}, Total Time: #{"%.2f" % metric.call_time}"
72
+ }.join("\n")
73
+ end
74
+
75
+ def at_limit?
76
+ @limit ||= config.value('database_metric_limit')
77
+ metrics.size >= @limit
78
+ end
79
+ end
80
+ end
@@ -0,0 +1,102 @@
1
+ module ScoutApm
2
+ class DbQueryMetricStats
3
+
4
+ DEFAULT_HISTOGRAM_SIZE = 50
5
+
6
+ attr_reader :model_name
7
+ attr_reader :operation
8
+ attr_reader :scope
9
+
10
+ attr_reader :transaction_count
11
+
12
+ attr_reader :call_count
13
+ attr_reader :call_time
14
+ attr_reader :rows_returned
15
+
16
+ attr_reader :min_call_time
17
+ attr_reader :max_call_time
18
+
19
+ attr_reader :min_rows_returned
20
+ attr_reader :max_rows_returned
21
+
22
+ attr_reader :histogram
23
+
24
+ def initialize(model_name, operation, scope, call_count, call_time, rows_returned)
25
+ @model_name = model_name
26
+ @operation = operation
27
+
28
+ @call_count = call_count
29
+
30
+ @call_time = call_time
31
+ @min_call_time = call_time
32
+ @max_call_time = call_time
33
+
34
+ @rows_returned = rows_returned
35
+ @min_rows_returned = rows_returned
36
+ @max_rows_returned = rows_returned
37
+
38
+ # Should we have a histogram for timing, and one for rows_returned?
39
+ # This histogram is for call_time
40
+ @histogram = NumericHistogram.new(DEFAULT_HISTOGRAM_SIZE)
41
+ @histogram.add(call_time)
42
+
43
+ @transaction_count = 0
44
+
45
+ @scope = scope
46
+ end
47
+
48
+ # Merge data in this scope. Used in DbQueryMetricSet
49
+ def key
50
+ @key ||= [model_name, operation, scope]
51
+ end
52
+
53
+ # Combine data from another DbQueryMetricStats into +self+. Modifies and returns +self+
54
+ def combine!(other)
55
+ return self if other == self
56
+
57
+ @transaction_count += other.transaction_count
58
+ @call_count += other.call_count
59
+ @rows_returned += other.rows_returned
60
+ @call_time += other.call_time
61
+
62
+ @min_call_time = other.min_call_time if @min_call_time.zero? or other.min_call_time < @min_call_time
63
+ @max_call_time = other.max_call_time if other.max_call_time > @max_call_time
64
+
65
+ @min_rows_returned = other.min_rows_returned if @min_rows_returned.zero? or other.min_rows_returned < @min_rows_returned
66
+ @max_rows_returned = other.max_rows_returned if other.max_rows_returned > @max_rows_returned
67
+
68
+ @histogram.combine!(other.histogram)
69
+ self
70
+ end
71
+
72
+ def as_json
73
+ json_attributes = [
74
+ :model_name,
75
+ :operation,
76
+ :scope,
77
+
78
+ :transaction_count,
79
+ :call_count,
80
+
81
+ :histogram,
82
+ :call_time,
83
+ :max_call_time,
84
+ :min_call_time,
85
+
86
+ :max_rows_returned,
87
+ :min_rows_returned,
88
+ :rows_returned,
89
+ ]
90
+
91
+ ScoutApm::AttributeArranger.call(self, json_attributes)
92
+ end
93
+
94
+ # Called by the Set on each DbQueryMetricStats object that it holds, only
95
+ # once during the recording of a transaction.
96
+ #
97
+ # Don't call elsewhere, and don't set to 1 in the initializer.
98
+ def increment_transaction_count!
99
+ @transaction_count += 1
100
+ end
101
+ end
102
+ end
@@ -17,6 +17,12 @@ module ScoutApm
17
17
  def track_one!(type, name, value, options={})
18
18
  end
19
19
 
20
+ def track_histograms!(histograms, options={})
21
+ end
22
+
23
+ def track_db_query_metrics!(db_query_metric_set, options={})
24
+ end
25
+
20
26
  def track_slow_transaction!(slow_transaction)
21
27
  end
22
28
 
@@ -224,7 +224,12 @@ module ScoutApm
224
224
  end
225
225
 
226
226
  def trace
227
- @trace ||= LayerConverters::SlowRequestConverter.new(tracked_request).call
227
+ @trace ||=
228
+ begin
229
+ layer_finder = LayerConverters::FindLayerByType.new(tracked_request)
230
+ converter = LayerConverters::SlowRequestConverter.new(tracked_request, layer_finder, ScoutApm::FakeStore.new)
231
+ converter.call
232
+ end
228
233
  end
229
234
 
230
235
  def payload
@@ -42,6 +42,23 @@ module ScoutApm
42
42
  end
43
43
  end
44
44
 
45
+ if Utils::KlassHelper.defined?("ActiveRecord::Base")
46
+ ::ActiveRecord::Base.class_eval do
47
+ include ::ScoutApm::Instruments::ActiveRecordUpdateInstruments
48
+ end
49
+ end
50
+
51
+ # Disabled until we can determine how to use Module#prepend in the
52
+ # agent. Otherwise, this will cause infinite loops if NewRelic is
53
+ # installed. We can't just use normal Module#include, since the
54
+ # original methods don't call super the way Base#save does
55
+ #
56
+ #if Utils::KlassHelper.defined?("ActiveRecord::Relation")
57
+ # ::ActiveRecord::Relation.class_eval do
58
+ # include ::ScoutApm::Instruments::ActiveRecordRelationInstruments
59
+ # end
60
+ #end
61
+
45
62
  if Utils::KlassHelper.defined?("ActiveRecord::Querying")
46
63
  ::ActiveRecord::Querying.module_eval do
47
64
  include ::ScoutApm::Tracer
@@ -208,5 +225,99 @@ module ScoutApm
208
225
  end
209
226
  end
210
227
  end
228
+
229
+ module ActiveRecordUpdateInstruments
230
+ def save(*args, &block)
231
+ model = self.class.name
232
+ operation = self.persisted? ? "Update" : "Create"
233
+
234
+ req = ScoutApm::RequestManager.lookup
235
+ layer = ScoutApm::Layer.new("ActiveRecord", Utils::ActiveRecordMetricName.new("", "#{model} #{operation}"))
236
+ req.start_layer(layer)
237
+ req.ignore_children!
238
+ begin
239
+ super(*args, &block)
240
+ ensure
241
+ req.acknowledge_children!
242
+ req.stop_layer
243
+ end
244
+ end
245
+
246
+ def save!(*args, &block)
247
+ model = self.class.name
248
+ operation = self.persisted? ? "Update" : "Create"
249
+
250
+ req = ScoutApm::RequestManager.lookup
251
+ layer = ScoutApm::Layer.new("ActiveRecord", Utils::ActiveRecordMetricName.new("", "#{model} #{operation}"))
252
+ req.start_layer(layer)
253
+ req.ignore_children!
254
+ begin
255
+ super(*args, &block)
256
+ ensure
257
+ req.acknowledge_children!
258
+ req.stop_layer
259
+ end
260
+ end
261
+ end
262
+
263
+ module ActiveRecordRelationInstruments
264
+ def self.included(instrumented_class)
265
+ ::ActiveRecord::Relation.class_eval do
266
+ alias_method :update_all_without_scout_instruments, :update_all
267
+ alias_method :update_all, :update_all_with_scout_instruments
268
+
269
+ alias_method :delete_all_without_scout_instruments, :delete_all
270
+ alias_method :delete_all, :delete_all_with_scout_instruments
271
+
272
+ alias_method :destroy_all_without_scout_instruments, :destroy_all
273
+ alias_method :destroy_all, :destroy_all_with_scout_instruments
274
+ end
275
+ end
276
+
277
+ def update_all_with_scout_instruments(*args, &block)
278
+ model = self.name
279
+
280
+ req = ScoutApm::RequestManager.lookup
281
+ layer = ScoutApm::Layer.new("ActiveRecord", Utils::ActiveRecordMetricName.new("", "#{model} Update"))
282
+ req.start_layer(layer)
283
+ req.ignore_children!
284
+ begin
285
+ update_all_without_scout_instruments(*args, &block)
286
+ ensure
287
+ req.acknowledge_children!
288
+ req.stop_layer
289
+ end
290
+ end
291
+
292
+ def delete_all_with_scout_instruments(*args, &block)
293
+ model = self.name
294
+
295
+ req = ScoutApm::RequestManager.lookup
296
+ layer = ScoutApm::Layer.new("ActiveRecord", Utils::ActiveRecordMetricName.new("", "#{model} Delete"))
297
+ req.start_layer(layer)
298
+ req.ignore_children!
299
+ begin
300
+ delete_all_without_scout_instruments(*args, &block)
301
+ ensure
302
+ req.acknowledge_children!
303
+ req.stop_layer
304
+ end
305
+ end
306
+
307
+ def destroy_all_with_scout_instruments(*args, &block)
308
+ model = self.name
309
+
310
+ req = ScoutApm::RequestManager.lookup
311
+ layer = ScoutApm::Layer.new("ActiveRecord", Utils::ActiveRecordMetricName.new("", "#{model} Delete"))
312
+ req.start_layer(layer)
313
+ req.ignore_children!
314
+ begin
315
+ destroy_all_without_scout_instruments(*args, &block)
316
+ ensure
317
+ req.acknowledge_children!
318
+ req.stop_layer
319
+ end
320
+ end
321
+ end
211
322
  end
212
323
  end