scout_apm 3.0.0.pre27 → 4.0.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (131) hide show
  1. checksums.yaml +5 -5
  2. data/.github/workflows/test.yml +49 -0
  3. data/.gitignore +1 -1
  4. data/.rubocop.yml +5 -5
  5. data/.travis.yml +19 -14
  6. data/CHANGELOG.markdown +139 -4
  7. data/Gemfile +1 -7
  8. data/README.markdown +13 -4
  9. data/Rakefile +1 -1
  10. data/ext/allocations/allocations.c +2 -0
  11. data/gems/README.md +28 -0
  12. data/gems/octoshark.gemfile +4 -0
  13. data/gems/rails3.gemfile +5 -0
  14. data/gems/rails4.gemfile +4 -0
  15. data/gems/rails5.gemfile +4 -0
  16. data/gems/rails6.gemfile +4 -0
  17. data/lib/scout_apm.rb +38 -9
  18. data/lib/scout_apm/agent.rb +29 -10
  19. data/lib/scout_apm/agent/exit_handler.rb +0 -1
  20. data/lib/scout_apm/agent_context.rb +22 -3
  21. data/lib/scout_apm/app_server_load.rb +7 -2
  22. data/lib/scout_apm/attribute_arranger.rb +0 -2
  23. data/lib/scout_apm/auto_instrument.rb +5 -0
  24. data/lib/scout_apm/auto_instrument/instruction_sequence.rb +31 -0
  25. data/lib/scout_apm/auto_instrument/layer.rb +23 -0
  26. data/lib/scout_apm/auto_instrument/parser.rb +27 -0
  27. data/lib/scout_apm/auto_instrument/rails.rb +175 -0
  28. data/lib/scout_apm/background_job_integrations/legacy_sneakers.rb +55 -0
  29. data/lib/scout_apm/background_job_integrations/que.rb +134 -0
  30. data/lib/scout_apm/background_job_integrations/resque.rb +6 -2
  31. data/lib/scout_apm/background_job_integrations/shoryuken.rb +124 -0
  32. data/lib/scout_apm/background_job_integrations/sidekiq.rb +5 -19
  33. data/lib/scout_apm/background_job_integrations/sneakers.rb +87 -0
  34. data/lib/scout_apm/config.rb +45 -8
  35. data/lib/scout_apm/detailed_trace.rb +217 -0
  36. data/lib/scout_apm/environment.rb +19 -1
  37. data/lib/scout_apm/error.rb +27 -0
  38. data/lib/scout_apm/error_service.rb +32 -0
  39. data/lib/scout_apm/error_service/error_buffer.rb +39 -0
  40. data/lib/scout_apm/error_service/error_record.rb +211 -0
  41. data/lib/scout_apm/error_service/ignored_exceptions.rb +66 -0
  42. data/lib/scout_apm/error_service/middleware.rb +32 -0
  43. data/lib/scout_apm/error_service/notifier.rb +33 -0
  44. data/lib/scout_apm/error_service/payload.rb +47 -0
  45. data/lib/scout_apm/error_service/periodic_work.rb +17 -0
  46. data/lib/scout_apm/error_service/railtie.rb +11 -0
  47. data/lib/scout_apm/error_service/sidekiq.rb +80 -0
  48. data/lib/scout_apm/extensions/transaction_callback_payload.rb +1 -1
  49. data/lib/scout_apm/fake_store.rb +3 -0
  50. data/lib/scout_apm/framework_integrations/rails_2.rb +2 -1
  51. data/lib/scout_apm/framework_integrations/rails_3_or_4.rb +3 -1
  52. data/lib/scout_apm/git_revision.rb +6 -3
  53. data/lib/scout_apm/instant/middleware.rb +2 -1
  54. data/lib/scout_apm/instrument_manager.rb +9 -7
  55. data/lib/scout_apm/instruments/action_controller_rails_2.rb +3 -1
  56. data/lib/scout_apm/instruments/action_controller_rails_3_rails4.rb +56 -55
  57. data/lib/scout_apm/instruments/action_view.rb +122 -26
  58. data/lib/scout_apm/instruments/active_record.rb +66 -18
  59. data/lib/scout_apm/instruments/http.rb +48 -0
  60. data/lib/scout_apm/instruments/memcached.rb +43 -0
  61. data/lib/scout_apm/instruments/mongoid.rb +9 -4
  62. data/lib/scout_apm/instruments/net_http.rb +8 -1
  63. data/lib/scout_apm/instruments/typhoeus.rb +88 -0
  64. data/lib/scout_apm/job_record.rb +4 -2
  65. data/lib/scout_apm/layaway_file.rb +4 -0
  66. data/lib/scout_apm/layer.rb +6 -57
  67. data/lib/scout_apm/layer_children_set.rb +9 -8
  68. data/lib/scout_apm/layer_converters/converter_base.rb +15 -30
  69. data/lib/scout_apm/layer_converters/slow_job_converter.rb +12 -2
  70. data/lib/scout_apm/layer_converters/slow_request_converter.rb +14 -4
  71. data/lib/scout_apm/layer_converters/trace_converter.rb +184 -0
  72. data/lib/scout_apm/limited_layer.rb +0 -7
  73. data/lib/scout_apm/metric_stats.rb +0 -8
  74. data/lib/scout_apm/middleware.rb +1 -1
  75. data/lib/scout_apm/periodic_work.rb +19 -0
  76. data/lib/scout_apm/remote/message.rb +4 -0
  77. data/lib/scout_apm/remote/server.rb +13 -1
  78. data/lib/scout_apm/reporter.rb +8 -3
  79. data/lib/scout_apm/reporting.rb +2 -1
  80. data/lib/scout_apm/request_histograms.rb +8 -0
  81. data/lib/scout_apm/serializers/app_server_load_serializer.rb +4 -0
  82. data/lib/scout_apm/serializers/directive_serializer.rb +4 -0
  83. data/lib/scout_apm/serializers/payload_serializer.rb +2 -2
  84. data/lib/scout_apm/serializers/payload_serializer_to_json.rb +30 -15
  85. data/lib/scout_apm/slow_job_record.rb +5 -1
  86. data/lib/scout_apm/slow_policy/age_policy.rb +33 -0
  87. data/lib/scout_apm/slow_policy/percent_policy.rb +22 -0
  88. data/lib/scout_apm/slow_policy/percentile_policy.rb +24 -0
  89. data/lib/scout_apm/slow_policy/policy.rb +21 -0
  90. data/lib/scout_apm/slow_policy/speed_policy.rb +16 -0
  91. data/lib/scout_apm/slow_request_policy.rb +18 -77
  92. data/lib/scout_apm/slow_transaction.rb +3 -1
  93. data/lib/scout_apm/store.rb +0 -1
  94. data/lib/scout_apm/tracked_request.rb +39 -30
  95. data/lib/scout_apm/utils/backtrace_parser.rb +3 -0
  96. data/lib/scout_apm/utils/marshal_logging.rb +90 -0
  97. data/lib/scout_apm/utils/sql_sanitizer.rb +10 -1
  98. data/lib/scout_apm/utils/sql_sanitizer_regex.rb +8 -1
  99. data/lib/scout_apm/utils/sql_sanitizer_regex_1_8_7.rb +6 -0
  100. data/lib/scout_apm/utils/unique_id.rb +27 -0
  101. data/lib/scout_apm/version.rb +1 -1
  102. data/scout_apm.gemspec +13 -7
  103. data/test/test_helper.rb +2 -2
  104. data/test/unit/agent_context_test.rb +29 -0
  105. data/test/unit/auto_instrument/assignments-instrumented.rb +31 -0
  106. data/test/unit/auto_instrument/assignments.rb +31 -0
  107. data/test/unit/auto_instrument/controller-ast.txt +57 -0
  108. data/test/unit/auto_instrument/controller-instrumented.rb +49 -0
  109. data/test/unit/auto_instrument/controller.rb +49 -0
  110. data/test/unit/auto_instrument/rescue_from-instrumented.rb +13 -0
  111. data/test/unit/auto_instrument/rescue_from.rb +13 -0
  112. data/test/unit/auto_instrument_test.rb +54 -0
  113. data/test/unit/environment_test.rb +2 -2
  114. data/test/unit/error_service/error_buffer_test.rb +25 -0
  115. data/test/unit/error_service/ignored_exceptions_test.rb +49 -0
  116. data/test/unit/instruments/active_record_test.rb +40 -0
  117. data/test/unit/layer_children_set_test.rb +9 -0
  118. data/test/unit/request_histograms_test.rb +17 -0
  119. data/test/unit/serializers/payload_serializer_test.rb +39 -5
  120. data/test/unit/slow_request_policy_test.rb +41 -13
  121. data/test/unit/sql_sanitizer_test.rb +78 -0
  122. metadata +101 -62
  123. data/ext/stacks/extconf.rb +0 -37
  124. data/ext/stacks/scout_atomics.h +0 -86
  125. data/ext/stacks/stacks.c +0 -814
  126. data/lib/scout_apm/slow_job_policy.rb +0 -111
  127. data/lib/scout_apm/trace_compactor.rb +0 -312
  128. data/lib/scout_apm/utils/fake_stacks.rb +0 -88
  129. data/test/unit/instruments/active_record_instruments_test.rb +0 -5
  130. data/test/unit/slow_job_policy_test.rb +0 -6
  131. data/tester.rb +0 -53
@@ -1,111 +0,0 @@
1
- # Long running class that determines if, and in how much detail a potentially
2
- # slow job should be recorded in
3
-
4
- module ScoutApm
5
- class SlowJobPolicy
6
- CAPTURE_TYPES = [
7
- CAPTURE_DETAIL = "capture_detail",
8
- CAPTURE_NONE = "capture_none",
9
- ]
10
-
11
- # Adjust speed points. See the function
12
- POINT_MULTIPLIER_SPEED = 0.25
13
-
14
- # For each minute we haven't seen an endpoint
15
- POINT_MULTIPLIER_AGE = 0.25
16
-
17
- # Outliers are worth up to "1000ms" of weight
18
- POINT_MULTIPLIER_PERCENTILE = 1.0
19
-
20
- # Points for an endpoint's who's throughput * response time is a large % of
21
- # overall time spent processing requests
22
- POINT_MULTIPLIER_PERCENT_TIME = 2.5
23
-
24
- # A hash of Job Names to the last time we stored a slow trace for it.
25
- #
26
- # Defaults to a start time that is pretty close to application boot time.
27
- # So the "age" of an endpoint we've never seen is the time the application
28
- # has been running.
29
- attr_reader :last_seen
30
-
31
- # The AgentContext we're running in
32
- attr_reader :context
33
-
34
- def initialize(context)
35
- @context = context
36
-
37
- zero_time = Time.now
38
- @last_seen = Hash.new { |h, k| h[k] = zero_time }
39
- end
40
-
41
- def stored!(request)
42
- last_seen[request.unique_name] = Time.now
43
- end
44
-
45
- # Determine if this job trace should be fully analyzed by scoring it
46
- # across several metrics, and then determining if that's good enough to
47
- # make it into this minute's payload.
48
- #
49
- # Due to the combining nature of the agent & layaway file, there's no
50
- # guarantee that a high scoring local champion will still be a winner when
51
- # they go up to "regionals" and are compared against the other processes
52
- # running on a node.
53
- def score(request)
54
- unique_name = request.unique_name
55
- if unique_name == :unknown
56
- return -1 # A negative score, should never be good enough to store.
57
- end
58
-
59
- total_time = request.root_layer.total_call_time
60
-
61
- # How long has it been since we've seen this?
62
- age = Time.now - last_seen[unique_name]
63
-
64
- # What approximate percentile was this request?
65
- percentile = context.request_histograms.approximate_quantile_of_value(unique_name, total_time)
66
-
67
- percent_of_total_time = context.transaction_time_consumed.percent_of_total(unique_name)
68
-
69
- return speed_points(total_time) + percentile_points(percentile) + age_points(age) + percent_time_points(percent_of_total_time)
70
- end
71
-
72
- private
73
-
74
- # Time in seconds
75
- # Logarithm keeps huge times from swamping the other metrics.
76
- # 1+ is necessary to keep the log function in positive territory.
77
- def speed_points(time)
78
- Math.log(1 + time) * POINT_MULTIPLIER_SPEED
79
- end
80
-
81
- def percentile_points(percentile)
82
- if percentile < 40
83
- 0.4 # Don't put much emphasis on capturing low percentiles.
84
- elsif percentile < 60
85
- 1.4 # Highest here to get mean traces
86
- elsif percentile < 90
87
- 0.7 # Between 60 & 90% is fine.
88
- elsif percentile >= 90
89
- 1.4 # Highest here to get 90+%ile traces
90
- else
91
- # impossible.
92
- percentile
93
- end
94
- end
95
-
96
- def age_points(age)
97
- age / 60.0 * POINT_MULTIPLIER_AGE
98
- end
99
-
100
- # Of the total time spent handling endpoints in this app, if this endpoint
101
- # is a higher percent, it should get more points.
102
- #
103
- # A: 20 calls @ 100ms each => 2 seconds of total time
104
- # B: 10 calls @ 100ms each => 1 second of total time
105
- #
106
- # Then A is 66% of the total call time
107
- def percent_time_points(percent) # Scale 0.0 - 1.0
108
- percent * POINT_MULTIPLIER_PERCENT_TIME
109
- end
110
- end
111
- end
@@ -1,312 +0,0 @@
1
- # Takes in a ton of traces. Structure is a several nested arrays:
2
- # [ # Traces
3
- # [ # Trace
4
- # [file,line,method,klass] # TraceLine (raw)
5
- # ]
6
- # ]
7
- #
8
- # Cleans them
9
- # Merges them by gem/app
10
- #
11
- module ScoutApm
12
- class TraceSet
13
- # A TraceCube object which is a glorified hash of { Trace -> Count }. Used to
14
- # collect up the count of each unique trace we've seen
15
- attr_reader :cube
16
-
17
- # Allow layer to push values in
18
- attr_accessor :raw_traces
19
- attr_accessor :skipped_in_gc
20
- attr_accessor :skipped_in_handler
21
- attr_accessor :skipped_in_job_registered
22
- attr_accessor :skipped_in_not_running
23
-
24
- def initialize
25
- @raw_traces = []
26
- @cube = TraceCube.new
27
- end
28
-
29
- # We need to know what the "Start" of this trace is. An untrimmed trace generally is:
30
- #
31
- # Gem
32
- # Gem
33
- # App
34
- # App
35
- # App <---- set root_class of this.
36
- # Rails
37
- # Rails
38
- def set_root_class(klass_name)
39
- @root_klass = klass_name.to_s
40
- end
41
-
42
- def to_a
43
- res = []
44
- create_cube!
45
- @cube.each do |(trace, count)|
46
- res << [trace.to_a, count]
47
- end
48
-
49
- res
50
- end
51
-
52
- def as_json
53
- res = []
54
- create_cube!
55
- @cube.each do |(trace, count)|
56
- res << [trace.as_json, count]
57
- end
58
-
59
- res
60
- end
61
-
62
- def create_cube!
63
- while raw_trace = @raw_traces.shift
64
- clean_trace = ScoutApm::CleanTrace.new(raw_trace, @root_klass)
65
- @cube << clean_trace
66
- end
67
- @raw_traces = []
68
- end
69
-
70
- def total_count
71
- create_cube!
72
- cube.inject(0) do |sum, (_, count)|
73
- sum + count
74
- end
75
- end
76
-
77
- def inspect
78
- create_cube!
79
- cube.map do |(trace, count)|
80
- "\t#{count} -- #{trace.first.klass}##{trace.first.method}\n\t\t#{trace.to_a[1].try(:klass)}##{trace.to_a[1].try(:method)}"
81
- end.join("\n")
82
- end
83
- end
84
-
85
- # A trace is a list of individual lines, where one called another, forming a backtrace.
86
- # Each line is made up of File, Line #, Klass, Method
87
- #
88
- # For the purpouses of this class:
89
- # "Top" of the trace means the currently-running method.
90
- # "Bottom" means the root of the call tree, from program start into rails and so on.
91
- #
92
- # This class trims off top and bottom to get a the meat of the trace
93
- class CleanTrace
94
- include Enumerable
95
-
96
- attr_reader :lines
97
-
98
- def initialize(raw_trace, root_klass=nil)
99
- @lines = Array(raw_trace).map {|frame, lineno| TraceLine.new(frame, lineno)}
100
- @root_klass = root_klass
101
-
102
- # A trace has interesting data in the middle of it, since normally it'll go
103
- # RailsCode -> App Code -> Gem Code.
104
- #
105
- # So we drop the code that leads up to your app, since a deep trace that
106
- # always says that you went through middleware and the rails router doesn't
107
- # help diagnose issues.
108
- drop_below_app
109
-
110
- # Then we drop most of the Gem Code, since you didn't write it, and in the
111
- # vast majority of the cases, the time spent there is because your app code
112
- # asked, not because of inherent issues with the gem. For instance, if you
113
- # fire off a slow query to a database gem, you probably want to be
114
- # optimizing the query, not trying to make the database gem faster.
115
- drop_above_app
116
- end
117
-
118
- # Iterate starting at END of array until a controller line is found. Pop off at that index - 1.
119
- def drop_below_app
120
- pops = 0
121
- index = lines.size - 1 # last index, not size.
122
-
123
- while index >= 0 && !lines[index].controller?(@root_klass)
124
- index -= 1
125
- pops += 1
126
- end
127
-
128
- lines.pop(pops)
129
- end
130
-
131
- # Find the closest mention of the application code from the currently-running method.
132
- # Then adjust by 1 if possible to capture the "first" line
133
- def drop_above_app
134
- ai = @lines.find_index(&:app?)
135
- if ai
136
- ai -= 1 if ai > 0
137
- @lines = @lines[ai .. -1]
138
- else
139
- @lines = [] # No app line in backtrace, nothing to show?
140
- end
141
- end
142
-
143
- def each
144
- @lines.each { |line| yield line }
145
- end
146
-
147
- def empty?
148
- @lines.empty?
149
- end
150
-
151
- def as_json
152
- @lines.map { |line| line.as_json }
153
- end
154
-
155
- ###############################
156
- # Hash Key interface
157
- def hash
158
- @lines.hash
159
- end
160
-
161
- def eql?(other)
162
- @lines.eql?(other.lines)
163
- end
164
- ###############################
165
- end
166
-
167
- class TraceLine
168
- # An opaque C object, only call Stacks#frame_* methods on this.
169
- attr_reader :frame
170
-
171
- # The line number. This doesn't appear to be obtainable from the frame itself
172
- attr_reader :lineno
173
-
174
- def initialize(frame, lineno)
175
- @frame = frame
176
- @lineno = lineno
177
- end
178
-
179
- # Returns the name of the last gem in the line
180
- def gem_name
181
- @gem_name ||= begin
182
- r = %r{\/gems/(.*?)/}.freeze
183
- results = file.scan(r)
184
- results[-1][0] # Scan will return a nested array, so extract out that nesting
185
- rescue
186
- nil
187
- end
188
- end
189
-
190
- def stdlib_name
191
- @stdlib_name ||= begin
192
- r = %r{#{Regexp.escape(RbConfig::TOPDIR)}/(.*?)}.freeze
193
- results = file.scan(r)
194
- results[-1][0] # Scan will return a nested array, so extract out that nesting
195
- rescue
196
- nil
197
- end
198
- end
199
-
200
- def file
201
- ScoutApm::Instruments::Stacks.frame_file(frame)
202
- end
203
-
204
-
205
- # If we ever want to get the "first line of the method" - ScoutApm::Instruments::Stacks.frame_lineno(frame)
206
- def line
207
- lineno
208
- end
209
-
210
- def klass
211
- ScoutApm::Instruments::Stacks.frame_klass(frame)
212
- end
213
-
214
- def method
215
- ScoutApm::Instruments::Stacks.frame_method(frame)
216
- end
217
-
218
- def gem?
219
- !!gem_name
220
- end
221
-
222
- def stdlib?
223
- !!stdlib_name
224
- end
225
-
226
- def app?
227
- r = %r|^#{Regexp.escape(ScoutApm::Environment.instance.root.to_s)}/|.freeze
228
- result = !gem_name && !stdlib_name && file =~ r
229
- !!result # coerce to a bool
230
- end
231
-
232
- def trim_file(file_path)
233
- return if file_path.nil?
234
- if gem?
235
- r = %r{.*gems/.*?/}.freeze
236
- file_path.sub(r, "/")
237
- elsif stdlib?
238
- file_path.sub(RbConfig::TOPDIR, '')
239
- elsif app?
240
- file_path.sub(ScoutApm::Environment.instance.root.to_s, '')
241
- end
242
- end
243
-
244
- # If root_klass is provided, just see if this is exactly that class. If not,
245
- # fall back on "is this in the app"
246
- def controller?(root_klass)
247
- return false if klass.nil? # main function doesn't have a file associated
248
-
249
- if root_klass
250
- klass == root_klass
251
- else
252
- app?
253
- end
254
- end
255
-
256
- def formatted_to_s
257
- "#{stdlib_name} #{klass}##{method} -- #{file}:#{line}"
258
- end
259
-
260
- def as_json
261
- [ trim_file(file), line, klass, method, app?, gem_name, stdlib_name ]
262
- end
263
-
264
- ###############################
265
- # Hash Key interface
266
-
267
- def hash
268
- # Note that this does not include line number. It caused a few situations
269
- # where we had a bunch of time spent in one method, but across a few lines,
270
- # we decided that it made sense to group them together.
271
- file.hash ^ method.hash ^ klass.hash
272
- end
273
-
274
- def eql?(other)
275
- file == other.file &&
276
- method == other.method &&
277
- klass == other.klass
278
- end
279
-
280
- ###############################
281
- end
282
-
283
- # Collects clean traces and counts how many of each we have.
284
- class TraceCube
285
- include Enumerable
286
-
287
- attr_reader :traces
288
- attr_reader :total_count
289
-
290
- def initialize
291
- @traces = Hash.new{ |h,k| h[k] = 0 }
292
- @total_count = 0
293
- end
294
-
295
- def <<(clean_trace)
296
- @total_count += 1
297
- @traces[clean_trace] += 1
298
- end
299
-
300
- # Yields two element array, the trace and the count of that trace
301
- # In descending order of count.
302
- def each
303
- @traces
304
- .to_a
305
- .each { |(trace, count)|
306
- next if trace.empty?
307
- yield [trace, count]
308
- }
309
- end
310
- end
311
- end
312
-
@@ -1,88 +0,0 @@
1
- # A fake implementation of the allocations native extension, for systems that don't support it.
2
- module ScoutApm
3
- module Instruments
4
- class Stacks
5
- ENABLED = false
6
- INSTALLED = false
7
-
8
- class << self
9
- def install(*args)
10
- # noop
11
- end
12
-
13
- def uninstall(*args)
14
- # noop
15
- end
16
-
17
- def start(*args)
18
- # noop
19
- end
20
-
21
- def add_profiled_thread(*args)
22
- # noop
23
- end
24
-
25
- def remove_profiled_thread(*args)
26
- # noop
27
- end
28
-
29
- def profile_frames(*args)
30
- [] # frames that were profiled (none)
31
- end
32
-
33
- def start_sampling(*args)
34
- # noop
35
- end
36
-
37
- def stop_sampling(*args)
38
- # noop
39
- end
40
-
41
- def update_indexes(*args)
42
- # noop
43
- true
44
- end
45
-
46
- def current_trace_index(*args)
47
- :opaque_value
48
- end
49
-
50
- def current_frame_index(*args)
51
- :opaque_value
52
- end
53
-
54
- def frame_klass(*args)
55
- nil
56
- end
57
-
58
- def frame_method(*args)
59
- nil
60
- end
61
-
62
- def frame_file(*args)
63
- nil
64
- end
65
-
66
- def frame_lineno(*args)
67
- nil
68
- end
69
-
70
- def skipped_in_gc(*args)
71
- 0
72
- end
73
-
74
- def skipped_in_handler(*args)
75
- 0
76
- end
77
-
78
- def skipped_in_job_registered(*args)
79
- 0
80
- end
81
-
82
- def skipped_in_not_running(*args)
83
- 0
84
- end
85
- end
86
- end
87
- end
88
- end