skylight 4.3.2 → 5.0.0.beta5

Sign up to get free protection for your applications and to get access to all the features.
Files changed (113) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +36 -3
  3. data/CONTRIBUTING.md +1 -7
  4. data/ext/extconf.rb +6 -5
  5. data/ext/libskylight.yml +7 -6
  6. data/ext/skylight_native.c +22 -99
  7. data/lib/skylight.rb +211 -14
  8. data/lib/skylight/api.rb +10 -3
  9. data/lib/skylight/cli.rb +4 -3
  10. data/lib/skylight/cli/doctor.rb +13 -14
  11. data/lib/skylight/cli/merger.rb +6 -4
  12. data/lib/skylight/config.rb +605 -127
  13. data/lib/skylight/deprecation.rb +17 -0
  14. data/lib/skylight/errors.rb +21 -6
  15. data/lib/skylight/extensions.rb +107 -0
  16. data/lib/skylight/extensions/source_location.rb +291 -0
  17. data/lib/skylight/formatters/http.rb +20 -0
  18. data/lib/skylight/gc.rb +109 -0
  19. data/lib/skylight/helpers.rb +52 -27
  20. data/lib/skylight/instrumenter.rb +326 -15
  21. data/lib/skylight/middleware.rb +138 -1
  22. data/lib/skylight/native.rb +52 -2
  23. data/lib/skylight/native_ext_fetcher.rb +4 -3
  24. data/lib/skylight/normalizers.rb +153 -0
  25. data/lib/skylight/normalizers/action_controller/process_action.rb +69 -0
  26. data/lib/skylight/normalizers/action_controller/send_file.rb +50 -0
  27. data/lib/skylight/normalizers/action_dispatch/process_middleware.rb +22 -0
  28. data/lib/skylight/normalizers/action_dispatch/route_set.rb +27 -0
  29. data/lib/skylight/normalizers/action_view/render_collection.rb +24 -0
  30. data/lib/skylight/normalizers/action_view/render_layout.rb +25 -0
  31. data/lib/skylight/normalizers/action_view/render_partial.rb +23 -0
  32. data/lib/skylight/normalizers/action_view/render_template.rb +23 -0
  33. data/lib/skylight/normalizers/active_job/perform.rb +86 -0
  34. data/lib/skylight/normalizers/active_model_serializers/render.rb +28 -0
  35. data/lib/skylight/normalizers/active_record/instantiation.rb +16 -0
  36. data/lib/skylight/normalizers/active_record/sql.rb +12 -0
  37. data/lib/skylight/normalizers/active_storage.rb +30 -0
  38. data/lib/skylight/normalizers/active_support/cache.rb +22 -0
  39. data/lib/skylight/normalizers/active_support/cache_clear.rb +16 -0
  40. data/lib/skylight/normalizers/active_support/cache_decrement.rb +16 -0
  41. data/lib/skylight/normalizers/active_support/cache_delete.rb +16 -0
  42. data/lib/skylight/normalizers/active_support/cache_exist.rb +16 -0
  43. data/lib/skylight/normalizers/active_support/cache_fetch_hit.rb +16 -0
  44. data/lib/skylight/normalizers/active_support/cache_generate.rb +16 -0
  45. data/lib/skylight/normalizers/active_support/cache_increment.rb +16 -0
  46. data/lib/skylight/normalizers/active_support/cache_read.rb +16 -0
  47. data/lib/skylight/normalizers/active_support/cache_read_multi.rb +16 -0
  48. data/lib/skylight/normalizers/active_support/cache_write.rb +16 -0
  49. data/lib/skylight/normalizers/coach/handler_finish.rb +46 -0
  50. data/lib/skylight/normalizers/coach/middleware_finish.rb +33 -0
  51. data/lib/skylight/normalizers/couch_potato/query.rb +20 -0
  52. data/lib/skylight/normalizers/data_mapper/sql.rb +12 -0
  53. data/lib/skylight/normalizers/default.rb +32 -0
  54. data/lib/skylight/normalizers/elasticsearch/request.rb +20 -0
  55. data/lib/skylight/normalizers/faraday/request.rb +40 -0
  56. data/lib/skylight/normalizers/grape/endpoint.rb +34 -0
  57. data/lib/skylight/normalizers/grape/endpoint_render.rb +25 -0
  58. data/lib/skylight/normalizers/grape/endpoint_run.rb +41 -0
  59. data/lib/skylight/normalizers/grape/endpoint_run_filters.rb +22 -0
  60. data/lib/skylight/normalizers/grape/format_response.rb +20 -0
  61. data/lib/skylight/normalizers/graphiti/render.rb +22 -0
  62. data/lib/skylight/normalizers/graphiti/resolve.rb +31 -0
  63. data/lib/skylight/normalizers/graphql/base.rb +132 -0
  64. data/lib/skylight/normalizers/render.rb +81 -0
  65. data/lib/skylight/normalizers/sequel/sql.rb +12 -0
  66. data/lib/skylight/normalizers/shrine.rb +34 -0
  67. data/lib/skylight/normalizers/sql.rb +45 -0
  68. data/lib/skylight/probes.rb +181 -0
  69. data/lib/skylight/probes/action_controller.rb +48 -0
  70. data/lib/skylight/probes/action_dispatch.rb +2 -0
  71. data/lib/skylight/probes/action_dispatch/request_id.rb +29 -0
  72. data/lib/skylight/probes/action_dispatch/routing/route_set.rb +28 -0
  73. data/lib/skylight/probes/action_view.rb +43 -0
  74. data/lib/skylight/probes/active_job.rb +27 -0
  75. data/lib/skylight/probes/active_job_enqueue.rb +41 -0
  76. data/lib/skylight/probes/active_model_serializers.rb +50 -0
  77. data/lib/skylight/probes/delayed_job.rb +149 -0
  78. data/lib/skylight/probes/elasticsearch.rb +38 -0
  79. data/lib/skylight/probes/excon.rb +25 -0
  80. data/lib/skylight/probes/excon/middleware.rb +66 -0
  81. data/lib/skylight/probes/faraday.rb +23 -0
  82. data/lib/skylight/probes/graphql.rb +43 -0
  83. data/lib/skylight/probes/httpclient.rb +44 -0
  84. data/lib/skylight/probes/middleware.rb +126 -0
  85. data/lib/skylight/probes/mongo.rb +164 -0
  86. data/lib/skylight/probes/mongoid.rb +13 -0
  87. data/lib/skylight/probes/net_http.rb +54 -0
  88. data/lib/skylight/probes/redis.rb +63 -0
  89. data/lib/skylight/probes/sequel.rb +33 -0
  90. data/lib/skylight/probes/sinatra.rb +63 -0
  91. data/lib/skylight/probes/sinatra_add_middleware.rb +10 -10
  92. data/lib/skylight/probes/tilt.rb +27 -0
  93. data/lib/skylight/railtie.rb +162 -18
  94. data/lib/skylight/sidekiq.rb +48 -0
  95. data/lib/skylight/subscriber.rb +110 -0
  96. data/lib/skylight/test.rb +146 -0
  97. data/lib/skylight/trace.rb +307 -10
  98. data/lib/skylight/user_config.rb +61 -0
  99. data/lib/skylight/util.rb +12 -0
  100. data/lib/skylight/util/allocation_free.rb +26 -0
  101. data/lib/skylight/util/clock.rb +56 -0
  102. data/lib/skylight/util/component.rb +5 -2
  103. data/lib/skylight/util/deploy.rb +7 -10
  104. data/lib/skylight/util/gzip.rb +20 -0
  105. data/lib/skylight/util/http.rb +4 -10
  106. data/lib/skylight/util/instrumenter_method.rb +26 -0
  107. data/lib/skylight/util/logging.rb +138 -0
  108. data/lib/skylight/util/lru_cache.rb +40 -0
  109. data/lib/skylight/util/platform.rb +1 -1
  110. data/lib/skylight/vendor/cli/thor/rake_compat.rb +1 -1
  111. data/lib/skylight/version.rb +5 -1
  112. data/lib/skylight/vm/gc.rb +68 -0
  113. metadata +114 -15
@@ -0,0 +1,17 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "active_support/deprecation"
4
+
5
+ module Skylight
6
+ SKYLIGHT_GEM_ROOT = "#{File.expand_path('../..', __dir__)}/"
7
+
8
+ class Deprecation < ActiveSupport::Deprecation
9
+ private
10
+
11
+ def ignored_callstack(path)
12
+ path.start_with?(SKYLIGHT_GEM_ROOT)
13
+ end
14
+ end
15
+
16
+ DEPRECATOR = Deprecation.new("6.0", "skylight")
17
+ end
@@ -1,4 +1,9 @@
1
+ require "json"
2
+
1
3
  module Skylight
4
+ # @api private
5
+ class ConfigError < RuntimeError; end
6
+
2
7
  class NativeError < StandardError
3
8
  @classes = {}
4
9
 
@@ -8,10 +13,10 @@ module Skylight
8
13
  end
9
14
 
10
15
  Skylight.module_eval <<-RUBY, __FILE__, __LINE__ + 1
11
- class #{name}Error < NativeError
12
- def self.code; #{code}; end
13
- def self.message; #{message.to_json}; end
14
- end
16
+ class #{name}Error < NativeError # class SqlLexError < NativeError
17
+ def self.code; #{code}; end # def self.code; 4; end
18
+ def self.message; #{message.to_json}; end # def self.message; "Failed to lex SQL query."; end
19
+ end # end
15
20
  RUBY
16
21
 
17
22
  klass = Skylight.const_get("#{name}Error")
@@ -29,13 +34,17 @@ module Skylight
29
34
  9999
30
35
  end
31
36
 
37
+ def self.formatted_code
38
+ format("%<code>04d", code: code)
39
+ end
40
+
32
41
  def self.message
33
42
  "Encountered an unknown internal error"
34
43
  end
35
44
 
36
45
  def initialize(method_name)
37
46
  @method_name = method_name
38
- super(format("[E%<code>04d] %<message>s [%<meth>s]", code: code, message: message, meth: method_name))
47
+ super(format("[E%<code>04d] %<message>s [%<meth>s]", code: code, message: self.class.message, meth: method_name))
39
48
  end
40
49
 
41
50
  def code
@@ -43,9 +52,12 @@ module Skylight
43
52
  end
44
53
 
45
54
  def formatted_code
46
- format("%04d", code)
55
+ self.class.formatted_code
47
56
  end
48
57
 
58
+ # E0002
59
+ # Too many unique descriptions - daemon only
60
+
49
61
  # E0003
50
62
  register(3, "MaximumTraceSpans", "Exceeded maximum number of spans in a trace.")
51
63
 
@@ -54,5 +66,8 @@ module Skylight
54
66
 
55
67
  # E0005
56
68
  register(5, "InstrumenterUnrecoverable", "Instrumenter is not running.")
69
+
70
+ # E0006
71
+ register(6, "InvalidUtf8", "Invalid UTF-8")
57
72
  end
58
73
  end
@@ -0,0 +1,107 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "active_support/inflector"
4
+
5
+ module Skylight
6
+ module Extensions
7
+ class Collection
8
+ def initialize(config, extensions = [])
9
+ @config = config
10
+ @extensions = extensions
11
+ end
12
+
13
+ def enable!(ext_name)
14
+ return if enabled?(ext_name)
15
+
16
+ find_by_name(ext_name) do |ext_class|
17
+ extensions << ext_class.new(config)
18
+ rememoize!
19
+ end
20
+ end
21
+
22
+ def disable!(ext_name)
23
+ find_by_name(ext_name) do |ext_class|
24
+ extensions.reject! { |x| x.is_a?(ext_class) }
25
+ rememoize!
26
+ end
27
+ end
28
+
29
+ def enabled?(ext_name)
30
+ return unless (ext_class = find_by_name(ext_name))
31
+
32
+ !!extensions.detect { |x| x.is_a?(ext_class) }
33
+ end
34
+
35
+ def process_trace_meta(meta)
36
+ extensions.each do |ext|
37
+ ext.process_trace_meta(meta)
38
+ end
39
+ end
40
+
41
+ # meta is a mutable hash that will be passed to the instrumenter.
42
+ # This method bridges Skylight.instrument and instrumenter.instrument.
43
+ def process_instrument_options(opts, meta)
44
+ extensions.each do |ext|
45
+ ext.process_instrument_options(opts, meta)
46
+ end
47
+ end
48
+
49
+ def process_normalizer_meta(payload, meta, **opts)
50
+ extensions.each do |ext|
51
+ ext.process_normalizer_meta(payload, meta, **opts)
52
+ end
53
+ end
54
+
55
+ def trace_preprocess_meta(meta)
56
+ extensions.each do |ext|
57
+ ext.trace_preprocess_meta(meta)
58
+ end
59
+ end
60
+
61
+ def allowed_meta_keys
62
+ @allowed_meta_keys ||= extensions.flat_map(&:allowed_meta_keys).uniq
63
+ end
64
+
65
+ private
66
+
67
+ attr_reader :extensions, :config
68
+
69
+ def find_by_name(ext_name)
70
+ begin
71
+ Skylight::Extensions.const_get(
72
+ ActiveSupport::Inflector.classify(ext_name)
73
+ )
74
+ rescue NameError
75
+ return nil
76
+ end.tap do |const|
77
+ yield const if block_given?
78
+ end
79
+ end
80
+
81
+ def rememoize!
82
+ @allowed_meta_keys = nil
83
+ allowed_meta_keys
84
+ end
85
+ end
86
+
87
+ class Extension
88
+ def initialize(config)
89
+ @config = config
90
+ end
91
+
92
+ def process_trace_meta(_meta); end
93
+
94
+ def process_instrument_options(_opts, _meta); end
95
+
96
+ def process_normalizer_meta(_payload, _meta, **opts); end
97
+
98
+ def trace_preprocess_meta(_meta); end
99
+
100
+ def allowed_meta_keys
101
+ []
102
+ end
103
+ end
104
+ end
105
+ end
106
+
107
+ require "skylight/extensions/source_location"
@@ -0,0 +1,291 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "skylight/util/lru_cache"
4
+ require "active_support/dependencies"
5
+
6
+ module Skylight
7
+ module Extensions
8
+ class SourceLocation < Extension
9
+ attr_reader :config
10
+
11
+ include Util::Logging
12
+
13
+ META_KEYS = %i[source_location source_file source_line].freeze
14
+ MAX_CALLER_DEPTH = 75
15
+
16
+ def initialize(*)
17
+ super
18
+ cache_size = (config[:source_location_cache_size] || 1000).to_i
19
+ @caller_cache = Util::LruCache.new(cache_size)
20
+ @instance_method_source_location_cache = Util::LruCache.new(cache_size)
21
+ gem_require_trie # memoize this at startup
22
+ end
23
+
24
+ def process_trace_meta(meta)
25
+ unless meta[:source_location] || meta[:source_file]
26
+ warn "Ignoring source_line without source_file" if meta[:source_line]
27
+ if (location = find_caller)
28
+ meta[:source_file] = location.absolute_path
29
+ meta[:source_line] = location.lineno
30
+ end
31
+ end
32
+ end
33
+
34
+ def process_instrument_options(opts, meta)
35
+ source_location = opts[:source_location] || opts[:meta]&.[](:source_location)
36
+ source_file = opts[:source_file] || opts[:meta]&.[](:source_file)
37
+ source_line = opts[:source_line] || opts[:meta]&.[](:source_line)
38
+ source_name_hint, const_name, method_name = opts[:source_location_hint] ||
39
+ opts[:meta]&.[](:source_location_hint)
40
+ instrument_location = opts[:sk_instrument_location]
41
+
42
+ if source_location
43
+ meta[:source_location] = source_location
44
+ elsif source_name_hint
45
+ source_location = dispatch_hinted_source_location(source_name_hint, const_name, method_name)
46
+ meta[:source_file], meta[:source_line] = source_location
47
+ meta.delete(:source_location_hint) if source_location
48
+ elsif source_file
49
+ meta[:source_file] = source_file
50
+ meta[:source_line] = source_line
51
+ elsif instrument_location && project_path?(instrument_location.absolute_path)
52
+ meta[:source_file] = instrument_location.absolute_path
53
+ meta[:source_line] = instrument_location.lineno
54
+ else
55
+ warn "Ignoring source_line without source_file" if source_line
56
+ if (location = find_caller(cache_key: opts.hash))
57
+ meta[:source_file] = location.absolute_path
58
+ meta[:source_line] = location.lineno
59
+ end
60
+ end
61
+
62
+ meta
63
+ end
64
+
65
+ def process_normalizer_meta(payload, meta, **opts)
66
+ if opts[:source_location] && (opts[:source_file] || opts[:source_line])
67
+ warn "Found both source_location and source_file or source_line in normalizer\n" \
68
+ " location=#{opts[:source_location]}; file=#{opts[:source_file]}; line=#{opts[:source_line]}"
69
+ end
70
+
71
+ sl =
72
+ if (source_name, constant_name, method_name = opts[:source_location_hint])
73
+ dispatch_hinted_source_location(
74
+ source_name,
75
+ constant_name,
76
+ method_name
77
+ )
78
+ elsif opts[:source_file]
79
+ [opts[:source_file], opts[:source_line]]
80
+ end
81
+
82
+ sl ||= source_location(payload, meta, cache_key: opts[:cache_key])
83
+
84
+ if sl
85
+ debug("normalizer source_location=#{sl}")
86
+ meta[:source_file], meta[:source_line] = sl
87
+ end
88
+
89
+ meta
90
+ end
91
+
92
+ def trace_preprocess_meta(meta)
93
+ source_line = meta.delete(:source_line)
94
+ source_file = meta.delete(:source_file)
95
+
96
+ if meta[:source_location]
97
+ if source_file || source_line
98
+ warn "Found both source_location and source_file or source_line, using source_location\n" \
99
+ " location=#{meta[:source_location]}; file=#{source_file}; line=#{source_line}"
100
+ end
101
+
102
+ unless meta[:source_location].is_a?(String)
103
+ warn "Found non-string value for source_location; skipping"
104
+ meta.delete(:source_location)
105
+ end
106
+ elsif source_file
107
+ meta[:source_location] = sanitize_source_location(source_file, source_line)
108
+ elsif source_line
109
+ warn "Ignoring source_line without source_file; source_line=#{source_line}"
110
+ end
111
+
112
+ if meta[:source_location]
113
+ debug("source_location=#{meta[:source_location]}")
114
+ end
115
+ end
116
+
117
+ def allowed_meta_keys
118
+ META_KEYS
119
+ end
120
+
121
+ protected
122
+
123
+ def dispatch_hinted_source_location(source_name, const_name, method_name)
124
+ return unless const_name && method_name
125
+
126
+ instance_method_source_location(const_name, method_name, source_name: source_name)
127
+ end
128
+
129
+ # from normalizers.rb
130
+ # Returns an array of file and line
131
+ def source_location(payload, meta, cache_key: nil)
132
+ # FIXME: what should precedence be?
133
+ if meta.is_a?(Hash) && meta[:source_location]
134
+ meta.delete(:source_location)
135
+ elsif payload.is_a?(Hash) && payload[:sk_source_location]
136
+ payload[:sk_source_location]
137
+ elsif (location = find_caller(cache_key: cache_key))
138
+ [location.absolute_path, location.lineno]
139
+ end
140
+ end
141
+
142
+ def find_caller(cache_key: nil)
143
+ # starting at 4 to skip Skylight extension processing logic
144
+ locations = ::Kernel.caller_locations(4..MAX_CALLER_DEPTH)
145
+
146
+ if cache_key
147
+ localized_cache_key = [cache_key, locations.map(&:lineno)].hash
148
+ @caller_cache.fetch(localized_cache_key) { find_caller_inner(locations) }
149
+ else
150
+ find_caller_inner(locations)
151
+ end
152
+ end
153
+
154
+ def project_path?(path)
155
+ return false unless path
156
+
157
+ # Must be in the project root
158
+ return false unless path.start_with?(config.root.to_s)
159
+ # Must not be Bundler's vendor location
160
+ return false if defined?(Bundler) && path.start_with?(Bundler.bundle_path.to_s)
161
+ # Must not be Ruby files
162
+ return false if path.include?("/ruby-#{RUBY_VERSION}/lib/ruby/")
163
+
164
+ # So it must be a project file
165
+ true
166
+ end
167
+
168
+ def instance_method_source_location(constant_name, method_name, source_name: :instance_method)
169
+ @instance_method_source_location_cache.fetch([constant_name, method_name, source_name]) do
170
+ if (constant = ::ActiveSupport::Dependencies.safe_constantize(constant_name))
171
+ if constant.instance_methods.include?(:"before_instrument_#{method_name}")
172
+ method_name = :"before_instrument_#{method_name}"
173
+ end
174
+ begin
175
+ unbound_method = case source_name
176
+ when :instance_method
177
+ find_instance_method(constant, method_name)
178
+ when :own_instance_method
179
+ find_own_instance_method(constant, method_name)
180
+ when :instance_method_super
181
+ find_instance_method_super(constant, method_name)
182
+ when :class_method
183
+ find_class_method(constant, method_name)
184
+ end
185
+
186
+ unbound_method&.source_location
187
+ rescue NameError
188
+ nil
189
+ end
190
+ end
191
+ end
192
+ end
193
+
194
+ def sanitize_source_location(path, line)
195
+ # Do this first since gems may be vendored in the app repo. However, it might be slower.
196
+ # Should we cache matches?
197
+ if (gem_name = find_source_gem(path))
198
+ path = gem_name
199
+ line = nil
200
+ elsif project_path?(path)
201
+ # Get relative path to root
202
+ path = Pathname.new(path).relative_path_from(config.root).to_s
203
+ else
204
+ return
205
+ end
206
+
207
+ line ? "#{path}:#{line}" : path
208
+ end
209
+
210
+ private
211
+
212
+ def gem_require_trie
213
+ @gem_require_trie ||= begin
214
+ trie = {}
215
+
216
+ Gem.loaded_specs.each do |name, spec|
217
+ next if config.source_location_ignored_gems&.include?(name)
218
+
219
+ spec.full_require_paths.each do |path|
220
+ t1 = trie
221
+
222
+ path.split(File::SEPARATOR).each do |segment|
223
+ t1[segment] ||= {}
224
+ t1 = t1[segment]
225
+ end
226
+
227
+ t1[:name] = name
228
+ end
229
+ end
230
+
231
+ trie
232
+ end
233
+ end
234
+
235
+ def find_source_gem(path)
236
+ return nil unless path
237
+
238
+ trie = gem_require_trie
239
+
240
+ path.split(File::SEPARATOR).each do |segment|
241
+ trie = trie[segment]
242
+ break unless trie
243
+ return trie[:name] if trie[:name]
244
+ end
245
+
246
+ nil
247
+ end
248
+
249
+ def find_caller_inner(locations)
250
+ # Start at file before this one
251
+ # NOTE: We could start farther back now to avoid more Skylight files
252
+ locations.find do |l|
253
+ absolute_path = l.absolute_path
254
+ find_source_gem(absolute_path) || project_path?(absolute_path)
255
+ end
256
+ end
257
+
258
+ # walks up the inheritance tree until it finds the last method
259
+ # without a super_method definition.
260
+ def find_instance_method_super(constant, method_name)
261
+ return unless (unbound_method = find_instance_method(constant, method_name))
262
+
263
+ while unbound_method.super_method
264
+ unbound_method = unbound_method.super_method
265
+ end
266
+
267
+ unbound_method
268
+ end
269
+
270
+ # walks up the inheritance tree until it finds the instance method
271
+ # belonging to the constant given (skip prepended modules)
272
+ def find_own_instance_method(constant, method_name)
273
+ return unless (unbound_method = find_instance_method(constant, method_name))
274
+
275
+ while unbound_method.owner != constant && unbound_method.super_method
276
+ unbound_method = unbound_method.super_method
277
+ end
278
+
279
+ unbound_method if unbound_method.owner == constant
280
+ end
281
+
282
+ def find_instance_method(constant, method_name)
283
+ constant.instance_method(method_name)
284
+ end
285
+
286
+ def find_class_method(constant, method_name)
287
+ constant.method(method_name)
288
+ end
289
+ end
290
+ end
291
+ end