skylight 4.2.1 → 5.0.0.beta

Sign up to get free protection for your applications and to get access to all the features.
Files changed (112) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +27 -0
  3. data/CONTRIBUTING.md +1 -7
  4. data/ext/extconf.rb +4 -3
  5. data/ext/libskylight.yml +5 -6
  6. data/ext/skylight_native.c +24 -100
  7. data/lib/skylight.rb +204 -14
  8. data/lib/skylight/api.rb +7 -3
  9. data/lib/skylight/cli.rb +4 -3
  10. data/lib/skylight/cli/doctor.rb +3 -2
  11. data/lib/skylight/cli/merger.rb +6 -4
  12. data/lib/skylight/config.rb +603 -126
  13. data/lib/skylight/deprecation.rb +15 -0
  14. data/lib/skylight/errors.rb +17 -2
  15. data/lib/skylight/extensions.rb +99 -0
  16. data/lib/skylight/extensions/source_location.rb +249 -0
  17. data/lib/skylight/fanout.rb +0 -0
  18. data/lib/skylight/formatters/http.rb +19 -0
  19. data/lib/skylight/gc.rb +109 -0
  20. data/lib/skylight/helpers.rb +18 -2
  21. data/lib/skylight/instrumenter.rb +325 -15
  22. data/lib/skylight/middleware.rb +138 -1
  23. data/lib/skylight/native.rb +51 -1
  24. data/lib/skylight/native_ext_fetcher.rb +2 -1
  25. data/lib/skylight/normalizers.rb +151 -0
  26. data/lib/skylight/normalizers/action_controller/process_action.rb +69 -0
  27. data/lib/skylight/normalizers/action_controller/send_file.rb +50 -0
  28. data/lib/skylight/normalizers/action_dispatch/process_middleware.rb +22 -0
  29. data/lib/skylight/normalizers/action_dispatch/route_set.rb +27 -0
  30. data/lib/skylight/normalizers/action_view/render_collection.rb +24 -0
  31. data/lib/skylight/normalizers/action_view/render_layout.rb +25 -0
  32. data/lib/skylight/normalizers/action_view/render_partial.rb +23 -0
  33. data/lib/skylight/normalizers/action_view/render_template.rb +23 -0
  34. data/lib/skylight/normalizers/active_job/perform.rb +81 -0
  35. data/lib/skylight/normalizers/active_model_serializers/render.rb +28 -0
  36. data/lib/skylight/normalizers/active_record/instantiation.rb +16 -0
  37. data/lib/skylight/normalizers/active_record/sql.rb +12 -0
  38. data/lib/skylight/normalizers/active_storage.rb +30 -0
  39. data/lib/skylight/normalizers/active_support/cache.rb +22 -0
  40. data/lib/skylight/normalizers/active_support/cache_clear.rb +16 -0
  41. data/lib/skylight/normalizers/active_support/cache_decrement.rb +16 -0
  42. data/lib/skylight/normalizers/active_support/cache_delete.rb +16 -0
  43. data/lib/skylight/normalizers/active_support/cache_exist.rb +16 -0
  44. data/lib/skylight/normalizers/active_support/cache_fetch_hit.rb +16 -0
  45. data/lib/skylight/normalizers/active_support/cache_generate.rb +16 -0
  46. data/lib/skylight/normalizers/active_support/cache_increment.rb +16 -0
  47. data/lib/skylight/normalizers/active_support/cache_read.rb +16 -0
  48. data/lib/skylight/normalizers/active_support/cache_read_multi.rb +16 -0
  49. data/lib/skylight/normalizers/active_support/cache_write.rb +16 -0
  50. data/lib/skylight/normalizers/coach/handler_finish.rb +46 -0
  51. data/lib/skylight/normalizers/coach/middleware_finish.rb +33 -0
  52. data/lib/skylight/normalizers/couch_potato/query.rb +20 -0
  53. data/lib/skylight/normalizers/data_mapper/sql.rb +12 -0
  54. data/lib/skylight/normalizers/default.rb +32 -0
  55. data/lib/skylight/normalizers/elasticsearch/request.rb +20 -0
  56. data/lib/skylight/normalizers/faraday/request.rb +40 -0
  57. data/lib/skylight/normalizers/grape/endpoint.rb +34 -0
  58. data/lib/skylight/normalizers/grape/endpoint_render.rb +25 -0
  59. data/lib/skylight/normalizers/grape/endpoint_run.rb +41 -0
  60. data/lib/skylight/normalizers/grape/endpoint_run_filters.rb +22 -0
  61. data/lib/skylight/normalizers/grape/format_response.rb +20 -0
  62. data/lib/skylight/normalizers/graphiti/render.rb +22 -0
  63. data/lib/skylight/normalizers/graphiti/resolve.rb +31 -0
  64. data/lib/skylight/normalizers/graphql/base.rb +131 -0
  65. data/lib/skylight/normalizers/render.rb +81 -0
  66. data/lib/skylight/normalizers/sequel/sql.rb +12 -0
  67. data/lib/skylight/normalizers/sql.rb +44 -0
  68. data/lib/skylight/probes.rb +153 -0
  69. data/lib/skylight/probes/action_controller.rb +48 -0
  70. data/lib/skylight/probes/action_dispatch.rb +2 -0
  71. data/lib/skylight/probes/action_dispatch/request_id.rb +29 -0
  72. data/lib/skylight/probes/action_dispatch/routing/route_set.rb +28 -0
  73. data/lib/skylight/probes/action_view.rb +43 -0
  74. data/lib/skylight/probes/active_job.rb +29 -0
  75. data/lib/skylight/probes/active_job_enqueue.rb +37 -0
  76. data/lib/skylight/probes/active_model_serializers.rb +54 -0
  77. data/lib/skylight/probes/delayed_job.rb +62 -0
  78. data/lib/skylight/probes/elasticsearch.rb +38 -0
  79. data/lib/skylight/probes/excon.rb +25 -0
  80. data/lib/skylight/probes/excon/middleware.rb +66 -0
  81. data/lib/skylight/probes/faraday.rb +23 -0
  82. data/lib/skylight/probes/graphql.rb +43 -0
  83. data/lib/skylight/probes/httpclient.rb +44 -0
  84. data/lib/skylight/probes/middleware.rb +125 -0
  85. data/lib/skylight/probes/mongo.rb +163 -0
  86. data/lib/skylight/probes/mongoid.rb +13 -0
  87. data/lib/skylight/probes/net_http.rb +55 -0
  88. data/lib/skylight/probes/redis.rb +60 -0
  89. data/lib/skylight/probes/sequel.rb +33 -0
  90. data/lib/skylight/probes/sinatra.rb +63 -0
  91. data/lib/skylight/probes/sinatra_add_middleware.rb +10 -10
  92. data/lib/skylight/probes/tilt.rb +27 -0
  93. data/lib/skylight/railtie.rb +162 -18
  94. data/lib/skylight/sidekiq.rb +43 -0
  95. data/lib/skylight/subscriber.rb +110 -0
  96. data/lib/skylight/test.rb +146 -0
  97. data/lib/skylight/trace.rb +301 -10
  98. data/lib/skylight/user_config.rb +61 -0
  99. data/lib/skylight/util.rb +12 -0
  100. data/lib/skylight/util/allocation_free.rb +26 -0
  101. data/lib/skylight/util/clock.rb +56 -0
  102. data/lib/skylight/util/component.rb +5 -2
  103. data/lib/skylight/util/deploy.rb +4 -4
  104. data/lib/skylight/util/gzip.rb +20 -0
  105. data/lib/skylight/util/http.rb +4 -10
  106. data/lib/skylight/util/instrumenter_method.rb +26 -0
  107. data/lib/skylight/util/logging.rb +138 -0
  108. data/lib/skylight/util/lru_cache.rb +42 -0
  109. data/lib/skylight/vendor/cli/thor/rake_compat.rb +1 -1
  110. data/lib/skylight/version.rb +5 -1
  111. data/lib/skylight/vm/gc.rb +68 -0
  112. metadata +116 -17
@@ -1,14 +1,9 @@
1
- require "skylight/core/railtie"
1
+ require "skylight"
2
+ require "rails"
2
3
 
3
4
  module Skylight
5
+ # @api private
4
6
  class Railtie < Rails::Railtie
5
- include Skylight::Core::Railtie
6
-
7
- # rubocop:disable Style/SingleLineMethods, Layout/EmptyLineBetweenDefs
8
- def self.config_class; Skylight::Config end
9
- def self.middleware_class; Skylight::Middleware end
10
- # rubocop:enable Style/SingleLineMethods, Layout/EmptyLineBetweenDefs
11
-
12
7
  config.skylight = ActiveSupport::OrderedOptions.new
13
8
 
14
9
  # The environments in which skylight should be enabled
@@ -33,12 +28,6 @@ module Skylight
33
28
 
34
29
  private
35
30
 
36
- def activate?(sk_config)
37
- return false unless super && sk_config
38
- show_worker_activation_warning(sk_config)
39
- true
40
- end
41
-
42
31
  # We must have an opt-in signal
43
32
  def show_worker_activation_warning(sk_config)
44
33
  reasons = []
@@ -51,16 +40,171 @@ module Skylight
51
40
  sk_config.logger.warn("Activating Skylight for Background Jobs because #{reasons.to_sentence}")
52
41
  end
53
42
 
43
+ def log_prefix
44
+ "[SKYLIGHT] [#{Skylight::VERSION}]"
45
+ end
46
+
54
47
  def development_warning
55
- super + "\n(To disable this message for all local apps, run `skylight disable_dev_warning`.)"
48
+ "#{log_prefix} Running Skylight in development mode. No data will be reported until you deploy your app.\n" \
49
+ "(To disable this message for all local apps, run `skylight disable_dev_warning`.)"
50
+ end
51
+
52
+ def run_initializer(app)
53
+ # Load probes even when agent is inactive to catch probe related bugs sooner
54
+ load_probes
55
+
56
+ config = load_skylight_config(app)
57
+
58
+ if activate?(config)
59
+ if config
60
+ if Skylight.start!(config)
61
+ set_middleware_position(app, config)
62
+ Rails.logger.info "#{log_prefix} Skylight agent enabled"
63
+ else
64
+ Rails.logger.info "#{log_prefix} Unable to start, see the Skylight logs for more details"
65
+ end
66
+ end
67
+ elsif Rails.env.development?
68
+ unless config.user_config.disable_dev_warning?
69
+ log_warning config, development_warning
70
+ end
71
+ elsif !Rails.env.test?
72
+ unless config.user_config.disable_env_warning?
73
+ log_warning config, "#{log_prefix} You are running in the #{Rails.env} environment but haven't added it " \
74
+ "to config.skylight.environments, so no data will be sent to Skylight servers."
75
+ end
76
+ end
77
+ rescue Skylight::ConfigError => e
78
+ Rails.logger.error "#{log_prefix} #{e.message}; disabling Skylight agent"
79
+ end
80
+
81
+ def log_warning(config, msg)
82
+ if config
83
+ config.alert_logger.warn(msg)
84
+ else
85
+ Rails.logger.warn(msg)
86
+ end
87
+ end
88
+
89
+ def existent_paths(paths)
90
+ paths.respond_to?(:existent) ? paths.existent : paths.select { |f| File.exist?(f) }
56
91
  end
57
92
 
58
93
  def load_skylight_config(app)
59
- super.tap do |sk_config|
60
- if sk_config && sk_config[:report_rails_env]
61
- sk_config[:env] ||= Rails.env.to_s
94
+ path = config_path(app)
95
+ path = nil unless File.exist?(path)
96
+
97
+ unless (tmp = app.config.paths["tmp"].first)
98
+ Rails.logger.error "#{log_prefix} tmp directory missing from rails configuration"
99
+ return nil
100
+ end
101
+
102
+ config = Config.load(file: path, environment: Rails.env.to_s)
103
+ config[:root] = Rails.root
104
+
105
+ configure_logging(config, app)
106
+
107
+ config[:'daemon.sockdir_path'] ||= tmp
108
+ config[:'normalizers.render.view_paths'] = existent_paths(app.config.paths["app/views"]) + [Rails.root.to_s]
109
+
110
+ if config[:report_rails_env]
111
+ config[:env] ||= Rails.env.to_s
112
+ end
113
+
114
+ config
115
+ end
116
+
117
+ def configure_logging(config, app)
118
+ if (logger = sk_rails_config(app).logger)
119
+ config.logger = logger
120
+ else
121
+ # Configure the log file destination
122
+ if (log_file = sk_rails_config(app).log_file)
123
+ config["log_file"] = log_file
62
124
  end
125
+
126
+ if (native_log_file = sk_rails_config(app).native_log_file)
127
+ config["native_log_file"] = native_log_file
128
+ end
129
+
130
+ if !config.key?("log_file") && !config.on_heroku?
131
+ config["log_file"] = File.join(Rails.root, "log/skylight.log")
132
+ end
133
+
134
+ # Configure the log level
135
+ if (level = sk_rails_config(app).log_level)
136
+ config["log_level"] = level
137
+ elsif !config.key?("log_level")
138
+ if (level = app.config.log_level)
139
+ config["log_level"] = level
140
+ end
141
+ end
142
+ end
143
+ end
144
+
145
+ def config_path(app)
146
+ File.expand_path(sk_rails_config.config_path, app.root)
147
+ end
148
+
149
+ def environments
150
+ Array(sk_rails_config.environments).map { |e| e&.to_s }.compact
151
+ end
152
+
153
+ def activate?(sk_config)
154
+ return false unless sk_config
155
+
156
+ key = "SKYLIGHT_ENABLED"
157
+ activate =
158
+ if ENV.key?(key)
159
+ ENV[key] !~ /^false$/i
160
+ else
161
+ environments.include?(Rails.env.to_s)
162
+ end
163
+
164
+ show_worker_activation_warning(sk_config) if activate
165
+
166
+ activate
167
+ end
168
+
169
+ def load_probes
170
+ probes = sk_rails_config.probes || []
171
+ Skylight::Probes.probe(*probes)
172
+ end
173
+
174
+ def middleware_position
175
+ if sk_rails_config.middleware_position.is_a?(Hash)
176
+ sk_rails_config.middleware_position.symbolize_keys
177
+ else
178
+ sk_rails_config.middleware_position
63
179
  end
64
180
  end
181
+
182
+ def insert_middleware(app, config)
183
+ if middleware_position.key?(:after)
184
+ app.middleware.insert_after(middleware_position[:after], Skylight::Middleware, config: config)
185
+ elsif middleware_position.key?(:before)
186
+ app.middleware.insert_before(middleware_position[:before], Skylight::Middleware, config: config)
187
+ else
188
+ raise "The middleware position you have set is invalid. Please be sure " \
189
+ "`config.skylight.middleware_position` is set up correctly."
190
+ end
191
+ end
192
+
193
+ def set_middleware_position(app, config)
194
+ if middleware_position.is_a?(Integer)
195
+ app.middleware.insert middleware_position, Skylight::Middleware, config: config
196
+ elsif middleware_position.is_a?(Hash) && middleware_position.keys.count == 1
197
+ insert_middleware(app, config)
198
+ elsif middleware_position.nil?
199
+ app.middleware.insert 0, Skylight::Middleware, config: config
200
+ else
201
+ raise "The middleware position you have set is invalid. Please be sure " \
202
+ "`config.skylight.middleware_position` is set up correctly."
203
+ end
204
+ end
205
+
206
+ def sk_rails_config(target = self)
207
+ target.config.skylight
208
+ end
65
209
  end
66
210
  end
@@ -0,0 +1,43 @@
1
+ module Skylight
2
+ module Sidekiq
3
+ def self.add_middleware
4
+ unless defined?(::Sidekiq)
5
+ Skylight.warn "Skylight for Sidekiq is active, but Sidekiq is not defined."
6
+ return
7
+ end
8
+
9
+ ::Sidekiq.configure_server do |sidekiq_config|
10
+ Skylight.debug "Adding Sidekiq Middleware"
11
+
12
+ sidekiq_config.server_middleware do |chain|
13
+ # Put it at the front
14
+ chain.prepend ServerMiddleware
15
+ end
16
+ end
17
+ end
18
+
19
+ class ServerMiddleware
20
+ include Util::Logging
21
+
22
+ def call(_worker, job, queue)
23
+ t { "Sidekiq middleware beginning trace" }
24
+ title = job["wrapped"] || job["class"]
25
+ Skylight.trace(title, "app.sidekiq.worker", title, segment: queue, component: :worker) do |trace|
26
+ begin
27
+ yield
28
+ rescue Exception # includes Sidekiq::Shutdown
29
+ trace.segment = "error" if trace
30
+ raise
31
+ end
32
+ end
33
+ end
34
+ end
35
+
36
+ ActiveSupport::Notifications.subscribe("started_instrumenter.skylight") \
37
+ do |_name, _started, _finished, _unique_id, payload|
38
+ if payload[:instrumenter].config.enable_sidekiq?
39
+ add_middleware
40
+ end
41
+ end
42
+ end
43
+ end
@@ -0,0 +1,110 @@
1
+ module Skylight
2
+ # @api private
3
+ class Subscriber
4
+ include Util::Logging
5
+
6
+ attr_reader :config, :normalizers
7
+
8
+ def initialize(config, instrumenter)
9
+ @config = config
10
+ @normalizers = Normalizers.build(config)
11
+ @instrumenter = instrumenter
12
+ @subscribers = []
13
+ end
14
+
15
+ def register!
16
+ unregister!
17
+ @normalizers.keys.each do |key|
18
+ @subscribers << ActiveSupport::Notifications.subscribe(key, self)
19
+ end
20
+ end
21
+
22
+ def unregister!
23
+ ActiveSupport::Notifications.unsubscribe @subscribers.shift until @subscribers.empty?
24
+ end
25
+
26
+ #
27
+ #
28
+ # ===== ActiveSupport::Notifications API
29
+ #
30
+ #
31
+
32
+ class Notification
33
+ attr_reader :name, :span
34
+
35
+ def initialize(name, span)
36
+ @name = name
37
+ @span = span
38
+ end
39
+ end
40
+
41
+ def start(name, _id, payload)
42
+ return if @instrumenter.disabled?
43
+ return unless (trace = @instrumenter.current_trace)
44
+
45
+ _start(trace, name, payload)
46
+ end
47
+
48
+ def finish(name, _id, payload)
49
+ return if @instrumenter.disabled?
50
+ return unless (trace = @instrumenter.current_trace)
51
+
52
+ while (curr = trace.notifications.pop)
53
+ next unless curr.name == name
54
+
55
+ meta = {}
56
+ meta[:exception] = payload[:exception] if payload[:exception]
57
+ meta[:exception_object] = payload[:exception_object] if payload[:exception_object]
58
+ trace.done(curr.span, meta) if curr.span
59
+ normalize_after(trace, curr.span, name, payload)
60
+ return
61
+ end
62
+ rescue Exception => e
63
+ error "Subscriber#finish error; msg=%s", e.message
64
+ debug "trace=%s", trace.inspect
65
+ debug "in: name=%s", name.inspect
66
+ debug "in: payload=%s", payload.inspect
67
+ t { e.backtrace.join("\n") }
68
+ nil
69
+ end
70
+
71
+ def publish(name, *args)
72
+ # Ignored for now because nothing in rails uses it
73
+ end
74
+
75
+ private
76
+
77
+ def normalize(*args)
78
+ @normalizers.normalize(*args)
79
+ end
80
+
81
+ def normalize_after(*args)
82
+ @normalizers.normalize_after(*args)
83
+ end
84
+
85
+ def _start(trace, name, payload)
86
+ result = normalize(trace, name, payload)
87
+
88
+ unless result == :skip
89
+ case result.size
90
+ when 3, 4
91
+ cat, title, desc, meta = result
92
+ else
93
+ raise "Invalid normalizer result: #{result.inspect}"
94
+ end
95
+
96
+ span = trace.instrument(cat, title, desc, meta)
97
+ end
98
+ rescue Exception => e
99
+ error "Subscriber#start error; msg=%s", e.message
100
+ debug "trace=%s", trace.inspect
101
+ debug "in: name=%s", name.inspect
102
+ debug "in: payload=%s", payload.inspect
103
+ debug "out: cat=%s, title=%s, desc=%s", cat.inspect, name.inspect, desc.inspect
104
+ t { e.backtrace.join("\n") }
105
+ nil
106
+ ensure
107
+ trace.notifications << Notification.new(name, span)
108
+ end
109
+ end
110
+ end
@@ -0,0 +1,146 @@
1
+ module Skylight
2
+ module Test
3
+ module Mocking
4
+ def mock!(config_opts = {}, &callback)
5
+ config_opts[:mock_submission] ||= callback || proc {}
6
+
7
+ config_class = Class.new(Config) do
8
+ def validate_with_server
9
+ true
10
+ end
11
+ end
12
+
13
+ config = config_class.load(config_opts)
14
+ config[:authentication] ||= "zomg"
15
+
16
+ class_eval do
17
+ unless const_defined?(:OriginalInstrumenter)
18
+ const_set :OriginalInstrumenter, Instrumenter
19
+ remove_const :Instrumenter
20
+ const_set(:Instrumenter, Class.new(OriginalInstrumenter) do
21
+ def self.name
22
+ "Mocked Instrumenter"
23
+ end
24
+
25
+ def self.native_new(*)
26
+ allocate
27
+ end
28
+
29
+ def native_start
30
+ true
31
+ end
32
+
33
+ def native_submit_trace(trace)
34
+ config[:mock_submission].call(trace)
35
+ end
36
+
37
+ def native_stop; end
38
+ end)
39
+
40
+ const_set :OriginalTrace, Trace
41
+ remove_const :Trace
42
+ const_set(:Trace, Class.new(OriginalTrace) do
43
+ def self.native_new(start, _uuid, endpoint, meta)
44
+ inst = allocate
45
+ inst.instance_variable_set(:@start, start)
46
+ inst.instance_variable_set(:@endpoint, endpoint)
47
+ inst.instance_variable_set(:@starting_endpoint, endpoint)
48
+ inst.instance_variable_set(:@meta, meta)
49
+ inst
50
+ end
51
+
52
+ attr_reader :endpoint, :starting_endpoint, :meta
53
+
54
+ def mock_spans
55
+ @mock_spans ||= []
56
+ end
57
+
58
+ def filter_spans
59
+ if block_given?
60
+ mock_spans.select { |span| yield span }
61
+ else
62
+ mock_spans.reject { |span| span[:cat] == "noise.gc" }
63
+ end
64
+ end
65
+
66
+ def native_get_uuid
67
+ @uuid
68
+ end
69
+
70
+ def uuid=(value)
71
+ @uuid = value
72
+ end
73
+
74
+ def native_get_started_at
75
+ @start
76
+ end
77
+
78
+ def native_set_endpoint(endpoint)
79
+ @endpoint = endpoint
80
+ end
81
+
82
+ def native_set_component(component)
83
+ @component = component
84
+ end
85
+
86
+ def native_start_span(time, cat)
87
+ span = {
88
+ start: time,
89
+ cat: cat
90
+ }
91
+ mock_spans << span
92
+ # Return integer like the native method does
93
+ mock_spans.index(span)
94
+ end
95
+
96
+ def native_span_set_title(span, title)
97
+ mock_spans[span][:title] = title
98
+ end
99
+
100
+ def native_span_set_description(span, desc)
101
+ mock_spans[span][:desc] = desc
102
+ end
103
+
104
+ def native_span_set_meta(span, meta)
105
+ mock_spans[span][:meta] = meta
106
+ end
107
+
108
+ def native_span_started(span); end
109
+
110
+ def native_span_set_exception(span, exception_object, exception)
111
+ mock_spans[span][:exception_object] = exception_object
112
+ mock_spans[span][:exception] = exception
113
+ end
114
+
115
+ def native_stop_span(span, time)
116
+ span = mock_spans[span]
117
+ span[:duration] = time - span[:start]
118
+ nil
119
+ end
120
+
121
+ def native_use_pruning
122
+ @using_native_pruning = true
123
+ end
124
+ end)
125
+ end
126
+ end
127
+
128
+ start!(config)
129
+ end
130
+
131
+ def unmock!
132
+ if const_defined?(:OriginalInstrumenter)
133
+ class_eval do
134
+ remove_const :Instrumenter
135
+ const_set :Instrumenter, OriginalInstrumenter
136
+ remove_const :OriginalInstrumenter
137
+
138
+ remove_const :Trace
139
+ const_set :Trace, OriginalTrace
140
+ remove_const :OriginalTrace
141
+ end
142
+ end
143
+ end
144
+ end
145
+ end
146
+ end