skylight 4.2.1 → 5.0.0.beta

Sign up to get free protection for your applications and to get access to all the features.
Files changed (112) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +27 -0
  3. data/CONTRIBUTING.md +1 -7
  4. data/ext/extconf.rb +4 -3
  5. data/ext/libskylight.yml +5 -6
  6. data/ext/skylight_native.c +24 -100
  7. data/lib/skylight.rb +204 -14
  8. data/lib/skylight/api.rb +7 -3
  9. data/lib/skylight/cli.rb +4 -3
  10. data/lib/skylight/cli/doctor.rb +3 -2
  11. data/lib/skylight/cli/merger.rb +6 -4
  12. data/lib/skylight/config.rb +603 -126
  13. data/lib/skylight/deprecation.rb +15 -0
  14. data/lib/skylight/errors.rb +17 -2
  15. data/lib/skylight/extensions.rb +99 -0
  16. data/lib/skylight/extensions/source_location.rb +249 -0
  17. data/lib/skylight/fanout.rb +0 -0
  18. data/lib/skylight/formatters/http.rb +19 -0
  19. data/lib/skylight/gc.rb +109 -0
  20. data/lib/skylight/helpers.rb +18 -2
  21. data/lib/skylight/instrumenter.rb +325 -15
  22. data/lib/skylight/middleware.rb +138 -1
  23. data/lib/skylight/native.rb +51 -1
  24. data/lib/skylight/native_ext_fetcher.rb +2 -1
  25. data/lib/skylight/normalizers.rb +151 -0
  26. data/lib/skylight/normalizers/action_controller/process_action.rb +69 -0
  27. data/lib/skylight/normalizers/action_controller/send_file.rb +50 -0
  28. data/lib/skylight/normalizers/action_dispatch/process_middleware.rb +22 -0
  29. data/lib/skylight/normalizers/action_dispatch/route_set.rb +27 -0
  30. data/lib/skylight/normalizers/action_view/render_collection.rb +24 -0
  31. data/lib/skylight/normalizers/action_view/render_layout.rb +25 -0
  32. data/lib/skylight/normalizers/action_view/render_partial.rb +23 -0
  33. data/lib/skylight/normalizers/action_view/render_template.rb +23 -0
  34. data/lib/skylight/normalizers/active_job/perform.rb +81 -0
  35. data/lib/skylight/normalizers/active_model_serializers/render.rb +28 -0
  36. data/lib/skylight/normalizers/active_record/instantiation.rb +16 -0
  37. data/lib/skylight/normalizers/active_record/sql.rb +12 -0
  38. data/lib/skylight/normalizers/active_storage.rb +30 -0
  39. data/lib/skylight/normalizers/active_support/cache.rb +22 -0
  40. data/lib/skylight/normalizers/active_support/cache_clear.rb +16 -0
  41. data/lib/skylight/normalizers/active_support/cache_decrement.rb +16 -0
  42. data/lib/skylight/normalizers/active_support/cache_delete.rb +16 -0
  43. data/lib/skylight/normalizers/active_support/cache_exist.rb +16 -0
  44. data/lib/skylight/normalizers/active_support/cache_fetch_hit.rb +16 -0
  45. data/lib/skylight/normalizers/active_support/cache_generate.rb +16 -0
  46. data/lib/skylight/normalizers/active_support/cache_increment.rb +16 -0
  47. data/lib/skylight/normalizers/active_support/cache_read.rb +16 -0
  48. data/lib/skylight/normalizers/active_support/cache_read_multi.rb +16 -0
  49. data/lib/skylight/normalizers/active_support/cache_write.rb +16 -0
  50. data/lib/skylight/normalizers/coach/handler_finish.rb +46 -0
  51. data/lib/skylight/normalizers/coach/middleware_finish.rb +33 -0
  52. data/lib/skylight/normalizers/couch_potato/query.rb +20 -0
  53. data/lib/skylight/normalizers/data_mapper/sql.rb +12 -0
  54. data/lib/skylight/normalizers/default.rb +32 -0
  55. data/lib/skylight/normalizers/elasticsearch/request.rb +20 -0
  56. data/lib/skylight/normalizers/faraday/request.rb +40 -0
  57. data/lib/skylight/normalizers/grape/endpoint.rb +34 -0
  58. data/lib/skylight/normalizers/grape/endpoint_render.rb +25 -0
  59. data/lib/skylight/normalizers/grape/endpoint_run.rb +41 -0
  60. data/lib/skylight/normalizers/grape/endpoint_run_filters.rb +22 -0
  61. data/lib/skylight/normalizers/grape/format_response.rb +20 -0
  62. data/lib/skylight/normalizers/graphiti/render.rb +22 -0
  63. data/lib/skylight/normalizers/graphiti/resolve.rb +31 -0
  64. data/lib/skylight/normalizers/graphql/base.rb +131 -0
  65. data/lib/skylight/normalizers/render.rb +81 -0
  66. data/lib/skylight/normalizers/sequel/sql.rb +12 -0
  67. data/lib/skylight/normalizers/sql.rb +44 -0
  68. data/lib/skylight/probes.rb +153 -0
  69. data/lib/skylight/probes/action_controller.rb +48 -0
  70. data/lib/skylight/probes/action_dispatch.rb +2 -0
  71. data/lib/skylight/probes/action_dispatch/request_id.rb +29 -0
  72. data/lib/skylight/probes/action_dispatch/routing/route_set.rb +28 -0
  73. data/lib/skylight/probes/action_view.rb +43 -0
  74. data/lib/skylight/probes/active_job.rb +29 -0
  75. data/lib/skylight/probes/active_job_enqueue.rb +37 -0
  76. data/lib/skylight/probes/active_model_serializers.rb +54 -0
  77. data/lib/skylight/probes/delayed_job.rb +62 -0
  78. data/lib/skylight/probes/elasticsearch.rb +38 -0
  79. data/lib/skylight/probes/excon.rb +25 -0
  80. data/lib/skylight/probes/excon/middleware.rb +66 -0
  81. data/lib/skylight/probes/faraday.rb +23 -0
  82. data/lib/skylight/probes/graphql.rb +43 -0
  83. data/lib/skylight/probes/httpclient.rb +44 -0
  84. data/lib/skylight/probes/middleware.rb +125 -0
  85. data/lib/skylight/probes/mongo.rb +163 -0
  86. data/lib/skylight/probes/mongoid.rb +13 -0
  87. data/lib/skylight/probes/net_http.rb +55 -0
  88. data/lib/skylight/probes/redis.rb +60 -0
  89. data/lib/skylight/probes/sequel.rb +33 -0
  90. data/lib/skylight/probes/sinatra.rb +63 -0
  91. data/lib/skylight/probes/sinatra_add_middleware.rb +10 -10
  92. data/lib/skylight/probes/tilt.rb +27 -0
  93. data/lib/skylight/railtie.rb +162 -18
  94. data/lib/skylight/sidekiq.rb +43 -0
  95. data/lib/skylight/subscriber.rb +110 -0
  96. data/lib/skylight/test.rb +146 -0
  97. data/lib/skylight/trace.rb +301 -10
  98. data/lib/skylight/user_config.rb +61 -0
  99. data/lib/skylight/util.rb +12 -0
  100. data/lib/skylight/util/allocation_free.rb +26 -0
  101. data/lib/skylight/util/clock.rb +56 -0
  102. data/lib/skylight/util/component.rb +5 -2
  103. data/lib/skylight/util/deploy.rb +4 -4
  104. data/lib/skylight/util/gzip.rb +20 -0
  105. data/lib/skylight/util/http.rb +4 -10
  106. data/lib/skylight/util/instrumenter_method.rb +26 -0
  107. data/lib/skylight/util/logging.rb +138 -0
  108. data/lib/skylight/util/lru_cache.rb +42 -0
  109. data/lib/skylight/vendor/cli/thor/rake_compat.rb +1 -1
  110. data/lib/skylight/version.rb +5 -1
  111. data/lib/skylight/vm/gc.rb +68 -0
  112. metadata +116 -17
@@ -0,0 +1,15 @@
1
+ require "active_support/deprecation"
2
+
3
+ module Skylight
4
+ SKYLIGHT_GEM_ROOT = File.expand_path("../..", __dir__) + "/"
5
+
6
+ class Deprecation < ActiveSupport::Deprecation
7
+ private
8
+
9
+ def ignored_callstack(path)
10
+ path.start_with?(SKYLIGHT_GEM_ROOT)
11
+ end
12
+ end
13
+
14
+ DEPRECATOR = Deprecation.new("6.0", "skylight")
15
+ end
@@ -1,4 +1,9 @@
1
+ require "json"
2
+
1
3
  module Skylight
4
+ # @api private
5
+ class ConfigError < RuntimeError; end
6
+
2
7
  class NativeError < StandardError
3
8
  @classes = {}
4
9
 
@@ -29,13 +34,17 @@ module Skylight
29
34
  9999
30
35
  end
31
36
 
37
+ def self.formatted_code
38
+ format("%<code>04d", code: code)
39
+ end
40
+
32
41
  def self.message
33
42
  "Encountered an unknown internal error"
34
43
  end
35
44
 
36
45
  def initialize(method_name)
37
46
  @method_name = method_name
38
- super(format("[E%<code>04d] %<message>s [%<meth>s]", code: code, message: message, meth: method_name))
47
+ super(format("[E%<code>04d] %<message>s [%<meth>s]", code: code, message: self.class.message, meth: method_name))
39
48
  end
40
49
 
41
50
  def code
@@ -43,9 +52,12 @@ module Skylight
43
52
  end
44
53
 
45
54
  def formatted_code
46
- format("%04d", code)
55
+ self.class.formatted_code
47
56
  end
48
57
 
58
+ # E0002
59
+ # Too many unique descriptions - daemon only
60
+
49
61
  # E0003
50
62
  register(3, "MaximumTraceSpans", "Exceeded maximum number of spans in a trace.")
51
63
 
@@ -54,5 +66,8 @@ module Skylight
54
66
 
55
67
  # E0005
56
68
  register(5, "InstrumenterUnrecoverable", "Instrumenter is not running.")
69
+
70
+ # E0006
71
+ register(6, "InvalidUtf8", "Invalid UTF-8")
57
72
  end
58
73
  end
@@ -0,0 +1,99 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "active_support/inflector"
4
+
5
+ module Skylight
6
+ module Extensions
7
+ class Collection
8
+ def initialize(config, extensions = [])
9
+ @config = config
10
+ @extensions = extensions
11
+ end
12
+
13
+ def enable!(ext_name)
14
+ return if enabled?(ext_name)
15
+
16
+ find_by_name(ext_name) do |ext_class|
17
+ extensions << ext_class.new(config)
18
+ rememoize!
19
+ end
20
+ end
21
+
22
+ def disable!(ext_name)
23
+ find_by_name(ext_name) do |ext_class|
24
+ extensions.reject! { |x| x.is_a?(ext_class) }
25
+ rememoize!
26
+ end
27
+ end
28
+
29
+ def enabled?(ext_name)
30
+ return unless (ext_class = find_by_name(ext_name))
31
+
32
+ !!extensions.detect { |x| x.is_a?(ext_class) }
33
+ end
34
+
35
+ # meta is a mutable hash that will be passed to the instrumenter.
36
+ # This method bridges Skylight.instrument and instrumenter.instrument.
37
+ def process_instrument_options(opts, meta)
38
+ extensions.each do |ext|
39
+ ext.process_instrument_options(opts, meta)
40
+ end
41
+ end
42
+
43
+ def process_normalizer_meta(payload, meta, **opts)
44
+ extensions.each do |ext|
45
+ ext.process_normalizer_meta(payload, meta, **opts)
46
+ end
47
+ end
48
+
49
+ def trace_preprocess_meta(meta)
50
+ extensions.each do |ext|
51
+ ext.trace_preprocess_meta(meta)
52
+ end
53
+ end
54
+
55
+ def allowed_meta_keys
56
+ @allowed_meta_keys ||= extensions.flat_map(&:allowed_meta_keys).uniq
57
+ end
58
+
59
+ private
60
+
61
+ attr_reader :extensions, :config
62
+
63
+ def find_by_name(ext_name)
64
+ begin
65
+ Skylight::Extensions.const_get(
66
+ ActiveSupport::Inflector.classify(ext_name)
67
+ )
68
+ rescue NameError
69
+ return nil
70
+ end.tap do |const|
71
+ yield const if block_given?
72
+ end
73
+ end
74
+
75
+ def rememoize!
76
+ @allowed_meta_keys = nil
77
+ allowed_meta_keys
78
+ end
79
+ end
80
+
81
+ class Extension
82
+ def initialize(config)
83
+ @config = config
84
+ end
85
+
86
+ def process_instrument_options(_opts, _meta); end
87
+
88
+ def process_normalizer_meta(_payload, _meta, **opts); end
89
+
90
+ def trace_preprocess_meta(_meta); end
91
+
92
+ def allowed_meta_keys
93
+ []
94
+ end
95
+ end
96
+ end
97
+ end
98
+
99
+ require "skylight/extensions/source_location"
@@ -0,0 +1,249 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "skylight/util/lru_cache"
4
+ require "active_support/dependencies"
5
+
6
+ module Skylight
7
+ module Extensions
8
+ class SourceLocation < Extension
9
+ attr_reader :config
10
+
11
+ include Util::Logging
12
+
13
+ META_KEYS = %i[source_location source_file source_line].freeze
14
+
15
+ def initialize(*)
16
+ super
17
+ @caller_cache = Util::LruCache.new(100)
18
+ @instance_method_source_location_cache = Util::LruCache.new(100)
19
+ gem_require_trie # memoize this at startup
20
+ end
21
+
22
+ def process_instrument_options(opts, meta)
23
+ source_location = opts[:source_location] || opts[:meta]&.[](:source_location)
24
+ source_file = opts[:source_file] || opts[:meta]&.[](:source_file)
25
+ source_line = opts[:source_line] || opts[:meta]&.[](:source_line)
26
+
27
+ if source_location
28
+ meta[:source_location] = source_location
29
+ elsif source_file
30
+ meta[:source_file] = source_file
31
+ meta[:source_line] = source_line
32
+ else
33
+ warn "Ignoring source_line without source_file" if source_line
34
+ if (location = find_caller(cache_key: opts.hash))
35
+ meta[:source_file] = location.absolute_path
36
+ meta[:source_line] = location.lineno
37
+ end
38
+ end
39
+
40
+ meta
41
+ end
42
+
43
+ def process_normalizer_meta(payload, meta, **opts)
44
+ sl = if ((source_name, *args) = opts[:source_location])
45
+ dispatch_hinted_source_location(
46
+ source_name,
47
+ payload,
48
+ meta,
49
+ args: args, **opts
50
+ )
51
+ end
52
+
53
+ sl ||= source_location(payload, meta, cache_key: opts[:cache_key])
54
+
55
+ if sl
56
+ debug("normalizer source_location=#{sl}")
57
+ meta[:source_file], meta[:source_line] = sl
58
+ end
59
+
60
+ meta
61
+ end
62
+
63
+ def trace_preprocess_meta(meta)
64
+ source_line = meta.delete(:source_line)
65
+ source_file = meta.delete(:source_file)
66
+
67
+ if meta[:source_location]
68
+ if source_file || source_line
69
+ warn "Found both source_location and source_file or source_line, using source_location\n" \
70
+ " location=#{meta[:source_location]}; file=#{source_file}; line=#{source_line}"
71
+ end
72
+
73
+ unless meta[:source_location].is_a?(String)
74
+ warn "Found non-string value for source_location; skipping"
75
+ meta.delete(:source_location)
76
+ end
77
+ elsif source_file
78
+ meta[:source_location] = sanitize_source_location(source_file, source_line)
79
+ elsif source_line
80
+ warn "Ignoring source_line without source_file; source_line=#{source_line}"
81
+ end
82
+
83
+ if meta[:source_location]
84
+ debug("source_location=#{meta[:source_location]}")
85
+ end
86
+ end
87
+
88
+ def allowed_meta_keys
89
+ META_KEYS
90
+ end
91
+
92
+ protected
93
+
94
+ def dispatch_hinted_source_location(source_name, payload, meta, args:, **opts)
95
+ const_name, method_name = args
96
+ return unless const_name && method_name
97
+
98
+ instance_method_source_location(const_name, method_name, source_name: source_name)
99
+ end
100
+
101
+ # from normalizers.rb
102
+ # Returns an array of file and line
103
+ def source_location(payload, meta, cache_key: nil)
104
+ # FIXME: what should precedence be?
105
+ if meta.is_a?(Hash) && meta[:source_location]
106
+ meta.delete(:source_location)
107
+ elsif payload.is_a?(Hash) && payload[:sk_source_location]
108
+ payload[:sk_source_location]
109
+ elsif (location = find_caller(cache_key: cache_key))
110
+ [location.absolute_path, location.lineno]
111
+ end
112
+ end
113
+
114
+ def find_caller(cache_key: nil)
115
+ if cache_key
116
+ @caller_cache.fetch(cache_key) { find_caller_inner }
117
+ else
118
+ find_caller_inner
119
+ end
120
+ end
121
+
122
+ def project_path?(path)
123
+ # Must be in the project root
124
+ return false unless path.start_with?(config.root.to_s)
125
+ # Must not be Bundler's vendor location
126
+ return false if defined?(Bundler) && path.start_with?(Bundler.bundle_path.to_s)
127
+ # Must not be Ruby files
128
+ return false if path.include?("/ruby-#{RUBY_VERSION}/lib/ruby/")
129
+
130
+ # So it must be a project file
131
+ true
132
+ end
133
+
134
+ def instance_method_source_location(constant_name, method_name, source_name: :instance_method)
135
+ @instance_method_source_location_cache.fetch([constant_name, method_name, source_name]) do
136
+ if (constant = ::ActiveSupport::Dependencies.safe_constantize(constant_name))
137
+ if constant.instance_methods.include?(:"before_instrument_#{method_name}")
138
+ method_name = :"before_instrument_#{method_name}"
139
+ end
140
+ begin
141
+ unbound_method = case source_name
142
+ when :instance_method
143
+ find_instance_method(constant, method_name)
144
+ when :own_instance_method
145
+ find_own_instance_method(constant, method_name)
146
+ when :instance_method_super
147
+ find_instance_method_super(constant, method_name)
148
+ end
149
+
150
+ unbound_method&.source_location
151
+ rescue NameError
152
+ nil
153
+ end
154
+ end
155
+ end
156
+ end
157
+
158
+ def sanitize_source_location(path, line)
159
+ # Do this first since gems may be vendored in the app repo. However, it might be slower.
160
+ # Should we cache matches?
161
+ if (gem_name = find_source_gem(path))
162
+ find_source_gem(path)
163
+ path = gem_name
164
+ line = nil
165
+ elsif project_path?(path)
166
+ # Get relative path to root
167
+ path = Pathname.new(path).relative_path_from(config.root).to_s
168
+ else
169
+ return
170
+ end
171
+
172
+ line ? "#{path}:#{line}" : path
173
+ end
174
+
175
+ private
176
+
177
+ def gem_require_trie
178
+ @gem_require_trie ||= begin
179
+ trie = {}
180
+
181
+ Gem.loaded_specs.each do |name, spec|
182
+ next if config.source_location_ignored_gems&.include?(name)
183
+
184
+ spec.full_require_paths.each do |path|
185
+ t1 = trie
186
+
187
+ path.split(File::SEPARATOR).each do |segment|
188
+ t1[segment] ||= {}
189
+ t1 = t1[segment]
190
+ end
191
+
192
+ t1[:name] = name
193
+ end
194
+ end
195
+
196
+ trie
197
+ end
198
+ end
199
+
200
+ def find_source_gem(path)
201
+ trie = gem_require_trie
202
+
203
+ path.split(File::SEPARATOR).each do |segment|
204
+ trie = trie[segment]
205
+ return unless trie
206
+ return trie[:name] if trie[:name]
207
+ end
208
+
209
+ nil
210
+ end
211
+
212
+ def find_caller_inner
213
+ # Start at file before this one
214
+ caller_locations(1).find do |l|
215
+ find_source_gem(l.absolute_path) || project_path?(l.absolute_path)
216
+ end
217
+ end
218
+
219
+ # walks up the inheritance tree until it finds the last method
220
+ # without a super_method definition.
221
+ def find_instance_method_super(constant, method_name)
222
+ return unless (unbound_method = find_instance_method(constant, method_name))
223
+
224
+ while unbound_method.super_method
225
+ unbound_method = unbound_method.super_method
226
+ end
227
+
228
+ unbound_method
229
+ end
230
+
231
+ # walks up the inheritance tree until it finds the instance method
232
+ # belonging to the constant given (skip prepended modules)
233
+ def find_own_instance_method(constant, method_name)
234
+ return unless (unbound_method = find_instance_method(constant, method_name))
235
+
236
+ while unbound_method.owner != constant && unbound_method.super_method
237
+ unbound_method = unbound_method.super_method
238
+ end
239
+
240
+ unbound_method if unbound_method.owner == constant
241
+ end
242
+
243
+ def find_instance_method(constant, method_name)
244
+ constant.instance_method(method_name)
245
+ end
246
+
247
+ end
248
+ end
249
+ end
File without changes
@@ -0,0 +1,19 @@
1
+ module Skylight
2
+ module Formatters
3
+ module HTTP
4
+ # Build instrumentation options for HTTP queries
5
+ #
6
+ # @param [String] method HTTP method, e.g. get, post
7
+ # @param [String] scheme HTTP scheme, e.g. http, https
8
+ # @param [String] host Request host, e.g. example.com
9
+ # @param [String, Integer] port Request port
10
+ # @param [String] path Request path
11
+ # @param [String] query Request query string
12
+ # @return [Hash] a hash containing `:category`, `:title`, and `:annotations`
13
+ def self.build_opts(method, _scheme, host, _port, _path, _query)
14
+ { category: "api.http.#{method.downcase}",
15
+ title: "#{method.upcase} #{host}" }
16
+ end
17
+ end
18
+ end
19
+ end
@@ -0,0 +1,109 @@
1
+ require "skylight/util/logging"
2
+
3
+ module Skylight
4
+ # @api private
5
+ class GC
6
+ METHODS = %i[enable total_time].freeze
7
+ TH_KEY = :SK_GC_CURR_WINDOW
8
+ MAX_COUNT = 1000
9
+ MAX_TIME = 30_000_000
10
+
11
+ include Util::Logging
12
+
13
+ attr_reader :config
14
+
15
+ def initialize(config, profiler)
16
+ @listeners = []
17
+ @config = config
18
+ @lock = Mutex.new
19
+ @time = 0
20
+
21
+ if METHODS.all? { |m| profiler.respond_to?(m) }
22
+ @profiler = profiler
23
+ @time = @profiler.total_time
24
+ else
25
+ debug "disabling GC profiling"
26
+ end
27
+ end
28
+
29
+ def enable
30
+ @profiler&.enable
31
+ end
32
+
33
+ # Total time in microseconds for GC over entire process lifetime
34
+ def total_time
35
+ @profiler ? @profiler.total_time : nil
36
+ end
37
+
38
+ def track
39
+ if @profiler
40
+ win = Window.new(self)
41
+
42
+ @lock.synchronize do
43
+ __update
44
+ @listeners << win
45
+
46
+ # Cleanup any listeners that might have leaked
47
+ @listeners.shift until @listeners[0].time < MAX_TIME
48
+
49
+ if @listeners.length > MAX_COUNT
50
+ @listeners.shift
51
+ end
52
+ end
53
+
54
+ win
55
+ else
56
+ Window.new(nil)
57
+ end
58
+ end
59
+
60
+ def release(win)
61
+ @lock.synchronize do
62
+ @listeners.delete(win)
63
+ end
64
+ end
65
+
66
+ def update
67
+ @lock.synchronize do
68
+ __update
69
+ end
70
+
71
+ nil
72
+ end
73
+
74
+ private
75
+
76
+ def __update
77
+ time = @profiler.total_time
78
+ diff = time - @time
79
+ @time = time
80
+
81
+ if diff > 0
82
+ @listeners.each do |l|
83
+ l.add(diff)
84
+ end
85
+ end
86
+ end
87
+
88
+ class Window
89
+ attr_reader :time
90
+
91
+ def initialize(global)
92
+ @global = global
93
+ @time = 0
94
+ end
95
+
96
+ def update
97
+ @global&.update
98
+ end
99
+
100
+ def add(time)
101
+ @time += time
102
+ end
103
+
104
+ def release
105
+ @global&.release(self)
106
+ end
107
+ end
108
+ end
109
+ end