skylight 4.3.2 → 5.0.0.beta5

Sign up to get free protection for your applications and to get access to all the features.
Files changed (113) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +36 -3
  3. data/CONTRIBUTING.md +1 -7
  4. data/ext/extconf.rb +6 -5
  5. data/ext/libskylight.yml +7 -6
  6. data/ext/skylight_native.c +22 -99
  7. data/lib/skylight.rb +211 -14
  8. data/lib/skylight/api.rb +10 -3
  9. data/lib/skylight/cli.rb +4 -3
  10. data/lib/skylight/cli/doctor.rb +13 -14
  11. data/lib/skylight/cli/merger.rb +6 -4
  12. data/lib/skylight/config.rb +605 -127
  13. data/lib/skylight/deprecation.rb +17 -0
  14. data/lib/skylight/errors.rb +21 -6
  15. data/lib/skylight/extensions.rb +107 -0
  16. data/lib/skylight/extensions/source_location.rb +291 -0
  17. data/lib/skylight/formatters/http.rb +20 -0
  18. data/lib/skylight/gc.rb +109 -0
  19. data/lib/skylight/helpers.rb +52 -27
  20. data/lib/skylight/instrumenter.rb +326 -15
  21. data/lib/skylight/middleware.rb +138 -1
  22. data/lib/skylight/native.rb +52 -2
  23. data/lib/skylight/native_ext_fetcher.rb +4 -3
  24. data/lib/skylight/normalizers.rb +153 -0
  25. data/lib/skylight/normalizers/action_controller/process_action.rb +69 -0
  26. data/lib/skylight/normalizers/action_controller/send_file.rb +50 -0
  27. data/lib/skylight/normalizers/action_dispatch/process_middleware.rb +22 -0
  28. data/lib/skylight/normalizers/action_dispatch/route_set.rb +27 -0
  29. data/lib/skylight/normalizers/action_view/render_collection.rb +24 -0
  30. data/lib/skylight/normalizers/action_view/render_layout.rb +25 -0
  31. data/lib/skylight/normalizers/action_view/render_partial.rb +23 -0
  32. data/lib/skylight/normalizers/action_view/render_template.rb +23 -0
  33. data/lib/skylight/normalizers/active_job/perform.rb +86 -0
  34. data/lib/skylight/normalizers/active_model_serializers/render.rb +28 -0
  35. data/lib/skylight/normalizers/active_record/instantiation.rb +16 -0
  36. data/lib/skylight/normalizers/active_record/sql.rb +12 -0
  37. data/lib/skylight/normalizers/active_storage.rb +30 -0
  38. data/lib/skylight/normalizers/active_support/cache.rb +22 -0
  39. data/lib/skylight/normalizers/active_support/cache_clear.rb +16 -0
  40. data/lib/skylight/normalizers/active_support/cache_decrement.rb +16 -0
  41. data/lib/skylight/normalizers/active_support/cache_delete.rb +16 -0
  42. data/lib/skylight/normalizers/active_support/cache_exist.rb +16 -0
  43. data/lib/skylight/normalizers/active_support/cache_fetch_hit.rb +16 -0
  44. data/lib/skylight/normalizers/active_support/cache_generate.rb +16 -0
  45. data/lib/skylight/normalizers/active_support/cache_increment.rb +16 -0
  46. data/lib/skylight/normalizers/active_support/cache_read.rb +16 -0
  47. data/lib/skylight/normalizers/active_support/cache_read_multi.rb +16 -0
  48. data/lib/skylight/normalizers/active_support/cache_write.rb +16 -0
  49. data/lib/skylight/normalizers/coach/handler_finish.rb +46 -0
  50. data/lib/skylight/normalizers/coach/middleware_finish.rb +33 -0
  51. data/lib/skylight/normalizers/couch_potato/query.rb +20 -0
  52. data/lib/skylight/normalizers/data_mapper/sql.rb +12 -0
  53. data/lib/skylight/normalizers/default.rb +32 -0
  54. data/lib/skylight/normalizers/elasticsearch/request.rb +20 -0
  55. data/lib/skylight/normalizers/faraday/request.rb +40 -0
  56. data/lib/skylight/normalizers/grape/endpoint.rb +34 -0
  57. data/lib/skylight/normalizers/grape/endpoint_render.rb +25 -0
  58. data/lib/skylight/normalizers/grape/endpoint_run.rb +41 -0
  59. data/lib/skylight/normalizers/grape/endpoint_run_filters.rb +22 -0
  60. data/lib/skylight/normalizers/grape/format_response.rb +20 -0
  61. data/lib/skylight/normalizers/graphiti/render.rb +22 -0
  62. data/lib/skylight/normalizers/graphiti/resolve.rb +31 -0
  63. data/lib/skylight/normalizers/graphql/base.rb +132 -0
  64. data/lib/skylight/normalizers/render.rb +81 -0
  65. data/lib/skylight/normalizers/sequel/sql.rb +12 -0
  66. data/lib/skylight/normalizers/shrine.rb +34 -0
  67. data/lib/skylight/normalizers/sql.rb +45 -0
  68. data/lib/skylight/probes.rb +181 -0
  69. data/lib/skylight/probes/action_controller.rb +48 -0
  70. data/lib/skylight/probes/action_dispatch.rb +2 -0
  71. data/lib/skylight/probes/action_dispatch/request_id.rb +29 -0
  72. data/lib/skylight/probes/action_dispatch/routing/route_set.rb +28 -0
  73. data/lib/skylight/probes/action_view.rb +43 -0
  74. data/lib/skylight/probes/active_job.rb +27 -0
  75. data/lib/skylight/probes/active_job_enqueue.rb +41 -0
  76. data/lib/skylight/probes/active_model_serializers.rb +50 -0
  77. data/lib/skylight/probes/delayed_job.rb +149 -0
  78. data/lib/skylight/probes/elasticsearch.rb +38 -0
  79. data/lib/skylight/probes/excon.rb +25 -0
  80. data/lib/skylight/probes/excon/middleware.rb +66 -0
  81. data/lib/skylight/probes/faraday.rb +23 -0
  82. data/lib/skylight/probes/graphql.rb +43 -0
  83. data/lib/skylight/probes/httpclient.rb +44 -0
  84. data/lib/skylight/probes/middleware.rb +126 -0
  85. data/lib/skylight/probes/mongo.rb +164 -0
  86. data/lib/skylight/probes/mongoid.rb +13 -0
  87. data/lib/skylight/probes/net_http.rb +54 -0
  88. data/lib/skylight/probes/redis.rb +63 -0
  89. data/lib/skylight/probes/sequel.rb +33 -0
  90. data/lib/skylight/probes/sinatra.rb +63 -0
  91. data/lib/skylight/probes/sinatra_add_middleware.rb +10 -10
  92. data/lib/skylight/probes/tilt.rb +27 -0
  93. data/lib/skylight/railtie.rb +162 -18
  94. data/lib/skylight/sidekiq.rb +48 -0
  95. data/lib/skylight/subscriber.rb +110 -0
  96. data/lib/skylight/test.rb +146 -0
  97. data/lib/skylight/trace.rb +307 -10
  98. data/lib/skylight/user_config.rb +61 -0
  99. data/lib/skylight/util.rb +12 -0
  100. data/lib/skylight/util/allocation_free.rb +26 -0
  101. data/lib/skylight/util/clock.rb +56 -0
  102. data/lib/skylight/util/component.rb +5 -2
  103. data/lib/skylight/util/deploy.rb +7 -10
  104. data/lib/skylight/util/gzip.rb +20 -0
  105. data/lib/skylight/util/http.rb +4 -10
  106. data/lib/skylight/util/instrumenter_method.rb +26 -0
  107. data/lib/skylight/util/logging.rb +138 -0
  108. data/lib/skylight/util/lru_cache.rb +40 -0
  109. data/lib/skylight/util/platform.rb +1 -1
  110. data/lib/skylight/vendor/cli/thor/rake_compat.rb +1 -1
  111. data/lib/skylight/version.rb +5 -1
  112. data/lib/skylight/vm/gc.rb +68 -0
  113. metadata +114 -15
@@ -0,0 +1,13 @@
1
+ module Skylight
2
+ module Probes
3
+ module Mongoid
4
+ class Probe
5
+ def install
6
+ Skylight::Probes.probe(:mongo)
7
+ end
8
+ end
9
+ end
10
+
11
+ register(:mongoid, "Mongoid", "mongoid", Mongoid::Probe.new)
12
+ end
13
+ end
@@ -0,0 +1,54 @@
1
+ require "skylight/formatters/http"
2
+
3
+ module Skylight
4
+ module Probes
5
+ module NetHTTP
6
+ module Instrumentation
7
+ def request(req, *)
8
+ return super if !started? || Probes::NetHTTP::Probe.disabled?
9
+
10
+ method = req.method
11
+
12
+ # req['host'] also includes special handling for default ports
13
+ host, port = req["host"] ? req["host"].split(":") : nil
14
+
15
+ # If we're connected with a persistent socket
16
+ host ||= address
17
+
18
+ path = req.path
19
+ scheme = use_ssl? ? "https" : "http"
20
+
21
+ # Contained in the path
22
+ query = nil
23
+
24
+ opts = Formatters::HTTP.build_opts(method, scheme, host, port, path, query)
25
+
26
+ Skylight.instrument(opts) { super }
27
+ end
28
+ end
29
+
30
+ # Probe for instrumenting Net::HTTP requests. Works by monkeypatching the default Net::HTTP#request method.
31
+ class Probe
32
+ DISABLED_KEY = :__skylight_net_http_disabled
33
+
34
+ def self.disable
35
+ state_was = Thread.current[DISABLED_KEY]
36
+ Thread.current[DISABLED_KEY] = true
37
+ yield
38
+ ensure
39
+ Thread.current[DISABLED_KEY] = state_was
40
+ end
41
+
42
+ def self.disabled?
43
+ !!Thread.current[DISABLED_KEY]
44
+ end
45
+
46
+ def install
47
+ Net::HTTP.prepend(Instrumentation)
48
+ end
49
+ end
50
+ end
51
+
52
+ register(:net_http, "Net::HTTP", "net/http", NetHTTP::Probe.new)
53
+ end
54
+ end
@@ -0,0 +1,63 @@
1
+ module Skylight
2
+ module Probes
3
+ module Redis
4
+ # Unfortunately, because of the nature of pipelining, there's no way for us to
5
+ # give a time breakdown on the individual items.
6
+
7
+ PIPELINED_OPTS = {
8
+ category: "db.redis.pipelined".freeze,
9
+ title: "PIPELINE".freeze,
10
+ internal: true
11
+ }.freeze
12
+
13
+ MULTI_OPTS = {
14
+ category: "db.redis.multi".freeze,
15
+ title: "MULTI".freeze,
16
+ internal: true
17
+ }.freeze
18
+
19
+ module ClientInstrumentation
20
+ def call(command, *)
21
+ command_name = command[0]
22
+
23
+ return super if command_name == :auth
24
+
25
+ opts = {
26
+ category: "db.redis.command",
27
+ title: command_name.upcase.to_s,
28
+ internal: true
29
+ }
30
+
31
+ Skylight.instrument(opts) { super }
32
+ end
33
+ end
34
+
35
+ module Instrumentation
36
+ def pipelined(*)
37
+ Skylight.instrument(PIPELINED_OPTS) { super }
38
+ end
39
+
40
+ def multi(*)
41
+ Skylight.instrument(MULTI_OPTS) { super }
42
+ end
43
+ end
44
+
45
+ class Probe
46
+ def install
47
+ version = defined?(::Redis::VERSION) ? Gem::Version.new(::Redis::VERSION) : nil
48
+
49
+ if !version || version < Gem::Version.new("3.0.0")
50
+ Skylight.error "The installed version of Redis doesn't support Middlewares. " \
51
+ "At least version 3.0.0 is required."
52
+ return
53
+ end
54
+
55
+ ::Redis::Client.prepend(ClientInstrumentation)
56
+ ::Redis.prepend(Instrumentation)
57
+ end
58
+ end
59
+ end
60
+
61
+ register(:redis, "Redis", "redis", Redis::Probe.new)
62
+ end
63
+ end
@@ -0,0 +1,33 @@
1
+ # Supports 3.12.0+
2
+ module Skylight
3
+ module Probes
4
+ module Sequel
5
+ class Probe
6
+ def install
7
+ require "sequel/database/logging"
8
+
9
+ method_name = ::Sequel::Database.method_defined?(:log_connection_yield) ? "log_connection_yield" : "log_yield"
10
+
11
+ mod = Module.new do
12
+ define_method method_name do |sql, *args, &block|
13
+ super(sql, *args) do
14
+ ::ActiveSupport::Notifications.instrument(
15
+ "sql.sequel",
16
+ sql: sql,
17
+ name: "SQL",
18
+ binds: args
19
+ ) do
20
+ block.call
21
+ end
22
+ end
23
+ end
24
+ end
25
+
26
+ ::Sequel::Database.prepend(mod)
27
+ end
28
+ end
29
+ end
30
+
31
+ register(:sequel, "Sequel", "sequel", Sequel::Probe.new)
32
+ end
33
+ end
@@ -0,0 +1,63 @@
1
+ module Skylight
2
+ module Probes
3
+ module Sinatra
4
+ module ClassInstrumentation
5
+ def compile!(verb, path, *)
6
+ super.tap do |_, _, keys_or_wrapper, wrapper|
7
+ wrapper ||= keys_or_wrapper
8
+
9
+ # Deal with the situation where the path is a regex, and the default behavior
10
+ # of Ruby stringification produces an unreadable mess
11
+ if path.is_a?(Regexp)
12
+ human_readable = "<sk-regex>%r{#{path.source}}</sk-regex>"
13
+ wrapper.instance_variable_set(:@route_name, "#{verb} #{human_readable}")
14
+ else
15
+ wrapper.instance_variable_set(:@route_name, "#{verb} #{path}")
16
+ end
17
+ end
18
+ end
19
+ end
20
+
21
+ module Instrumentation
22
+ def dispatch!(*)
23
+ super.tap do
24
+ if (trace = Skylight.instrumenter&.current_trace) && (route = env["sinatra.route"])
25
+ # Include the app's mount point (if available)
26
+ script_name = trace.instrumenter.config.sinatra_route_prefixes? && env["SCRIPT_NAME"]
27
+
28
+ trace.endpoint =
29
+ if script_name && !script_name.empty?
30
+ verb, path = route.split(" ", 2)
31
+ "#{verb} [#{script_name}]#{path}"
32
+ else
33
+ route
34
+ end
35
+ end
36
+ end
37
+ end
38
+
39
+ def compile_template(engine, data, options, *)
40
+ # Pass along a useful "virtual path" to Tilt. The Tilt probe will handle
41
+ # instrumenting correctly.
42
+ options[:sky_virtual_path] = data.is_a?(Symbol) ? data.to_s : "Inline template (#{engine})"
43
+
44
+ super
45
+ end
46
+ end
47
+
48
+ class Probe
49
+ def install
50
+ if ::Sinatra::VERSION < "1.4.0"
51
+ Skylight.error "Sinatra must be version 1.4.0 or greater."
52
+ return
53
+ end
54
+
55
+ ::Sinatra::Base.singleton_class.prepend(ClassInstrumentation)
56
+ ::Sinatra::Base.prepend(Instrumentation)
57
+ end
58
+ end
59
+ end
60
+
61
+ register(:sinatra, "Sinatra::Base", "sinatra/base", Sinatra::Probe.new)
62
+ end
63
+ end
@@ -1,20 +1,20 @@
1
1
  module Skylight
2
2
  module Probes
3
3
  module Sinatra
4
- class Probe
5
- def install
6
- class << ::Sinatra::Base
7
- alias_method :build_without_sk, :build
4
+ module Instrumentation
5
+ def build(*)
6
+ use Skylight::Middleware
7
+ super
8
+ end
9
+ end
8
10
 
9
- def build(*args, &block)
10
- use Skylight::Middleware
11
- build_without_sk(*args, &block)
12
- end
13
- end
11
+ class AddMiddlewareProbe
12
+ def install
13
+ ::Sinatra::Base.singleton_class.prepend(Instrumentation)
14
14
  end
15
15
  end
16
16
  end
17
17
 
18
- Skylight::Core::Probes.register(:sinatra_add_middleware, "Sinatra::Base", "sinatra/base", Sinatra::Probe.new)
18
+ register(:sinatra_add_middleware, "Sinatra::Base", "sinatra/base", Sinatra::AddMiddlewareProbe.new)
19
19
  end
20
20
  end
@@ -0,0 +1,27 @@
1
+ # Should support 0.2+, though not tested against older versions
2
+ module Skylight
3
+ module Probes
4
+ module Tilt
5
+ module Instrumentation
6
+ def render(*args, &block)
7
+ opts = {
8
+ category: "view.render.template",
9
+ title: options[:sky_virtual_path] || basename || "Unknown template name"
10
+ }
11
+
12
+ Skylight.instrument(opts) do
13
+ super(*args, &block)
14
+ end
15
+ end
16
+ end
17
+
18
+ class Probe
19
+ def install
20
+ ::Tilt::Template.prepend(Instrumentation)
21
+ end
22
+ end
23
+ end
24
+
25
+ register(:tilt, "Tilt::Template", "tilt/template", Tilt::Probe.new)
26
+ end
27
+ end
@@ -1,14 +1,9 @@
1
- require "skylight/core/railtie"
1
+ require "skylight"
2
+ require "rails"
2
3
 
3
4
  module Skylight
5
+ # @api private
4
6
  class Railtie < Rails::Railtie
5
- include Skylight::Core::Railtie
6
-
7
- # rubocop:disable Style/SingleLineMethods, Layout/EmptyLineBetweenDefs
8
- def self.config_class; Skylight::Config end
9
- def self.middleware_class; Skylight::Middleware end
10
- # rubocop:enable Style/SingleLineMethods, Layout/EmptyLineBetweenDefs
11
-
12
7
  config.skylight = ActiveSupport::OrderedOptions.new
13
8
 
14
9
  # The environments in which skylight should be enabled
@@ -33,12 +28,6 @@ module Skylight
33
28
 
34
29
  private
35
30
 
36
- def activate?(sk_config)
37
- return false unless super && sk_config
38
- show_worker_activation_warning(sk_config)
39
- true
40
- end
41
-
42
31
  # We must have an opt-in signal
43
32
  def show_worker_activation_warning(sk_config)
44
33
  reasons = []
@@ -51,16 +40,171 @@ module Skylight
51
40
  sk_config.logger.warn("Activating Skylight for Background Jobs because #{reasons.to_sentence}")
52
41
  end
53
42
 
43
+ def log_prefix
44
+ "[SKYLIGHT] [#{Skylight::VERSION}]"
45
+ end
46
+
54
47
  def development_warning
55
- super + "\n(To disable this message for all local apps, run `skylight disable_dev_warning`.)"
48
+ "#{log_prefix} Running Skylight in development mode. No data will be reported until you deploy your app.\n" \
49
+ "(To disable this message for all local apps, run `skylight disable_dev_warning`.)"
50
+ end
51
+
52
+ def run_initializer(app)
53
+ # Load probes even when agent is inactive to catch probe related bugs sooner
54
+ load_probes
55
+
56
+ config = load_skylight_config(app)
57
+
58
+ if activate?(config)
59
+ if config
60
+ if Skylight.start!(config)
61
+ set_middleware_position(app, config)
62
+ Rails.logger.info "#{log_prefix} Skylight agent enabled"
63
+ else
64
+ Rails.logger.info "#{log_prefix} Unable to start, see the Skylight logs for more details"
65
+ end
66
+ end
67
+ elsif Rails.env.development?
68
+ unless config.user_config.disable_dev_warning?
69
+ log_warning config, development_warning
70
+ end
71
+ elsif !Rails.env.test?
72
+ unless config.user_config.disable_env_warning?
73
+ log_warning config, "#{log_prefix} You are running in the #{Rails.env} environment but haven't added it " \
74
+ "to config.skylight.environments, so no data will be sent to Skylight servers."
75
+ end
76
+ end
77
+ rescue Skylight::ConfigError => e
78
+ Rails.logger.error "#{log_prefix} #{e.message}; disabling Skylight agent"
79
+ end
80
+
81
+ def log_warning(config, msg)
82
+ if config
83
+ config.alert_logger.warn(msg)
84
+ else
85
+ Rails.logger.warn(msg)
86
+ end
87
+ end
88
+
89
+ def existent_paths(paths)
90
+ paths.respond_to?(:existent) ? paths.existent : paths.select { |f| File.exist?(f) }
56
91
  end
57
92
 
58
93
  def load_skylight_config(app)
59
- super.tap do |sk_config|
60
- if sk_config && sk_config[:report_rails_env]
61
- sk_config[:env] ||= Rails.env.to_s
94
+ path = config_path(app)
95
+ path = nil unless File.exist?(path)
96
+
97
+ unless (tmp = app.config.paths["tmp"].first)
98
+ Rails.logger.error "#{log_prefix} tmp directory missing from rails configuration"
99
+ return nil
100
+ end
101
+
102
+ config = Config.load(file: path, priority_key: Rails.env.to_s)
103
+ config[:root] = Rails.root
104
+
105
+ configure_logging(config, app)
106
+
107
+ config[:'daemon.sockdir_path'] ||= tmp
108
+ config[:'normalizers.render.view_paths'] = existent_paths(app.config.paths["app/views"]) + [Rails.root.to_s]
109
+
110
+ if config[:report_rails_env]
111
+ config[:env] ||= Rails.env.to_s
112
+ end
113
+
114
+ config
115
+ end
116
+
117
+ def configure_logging(config, app)
118
+ if (logger = sk_rails_config(app).logger)
119
+ config.logger = logger
120
+ else
121
+ # Configure the log file destination
122
+ if (log_file = sk_rails_config(app).log_file)
123
+ config["log_file"] = log_file
62
124
  end
125
+
126
+ if (native_log_file = sk_rails_config(app).native_log_file)
127
+ config["native_log_file"] = native_log_file
128
+ end
129
+
130
+ if !config.key?("log_file") && !config.on_heroku?
131
+ config["log_file"] = File.join(Rails.root, "log/skylight.log")
132
+ end
133
+
134
+ # Configure the log level
135
+ if (level = sk_rails_config(app).log_level)
136
+ config["log_level"] = level
137
+ elsif !config.key?("log_level")
138
+ if (level = app.config.log_level)
139
+ config["log_level"] = level
140
+ end
141
+ end
142
+ end
143
+ end
144
+
145
+ def config_path(app)
146
+ File.expand_path(sk_rails_config.config_path, app.root)
147
+ end
148
+
149
+ def environments
150
+ Array(sk_rails_config.environments).map { |e| e&.to_s }.compact
151
+ end
152
+
153
+ def activate?(sk_config)
154
+ return false unless sk_config
155
+
156
+ key = "SKYLIGHT_ENABLED"
157
+ activate =
158
+ if ENV.key?(key)
159
+ ENV[key] !~ /^false$/i
160
+ else
161
+ environments.include?(Rails.env.to_s)
162
+ end
163
+
164
+ show_worker_activation_warning(sk_config) if activate
165
+
166
+ activate
167
+ end
168
+
169
+ def load_probes
170
+ probes = sk_rails_config.probes || []
171
+ Skylight::Probes.probe(*probes)
172
+ end
173
+
174
+ def middleware_position
175
+ if sk_rails_config.middleware_position.is_a?(Hash)
176
+ sk_rails_config.middleware_position.symbolize_keys
177
+ else
178
+ sk_rails_config.middleware_position
63
179
  end
64
180
  end
181
+
182
+ def insert_middleware(app, config)
183
+ if middleware_position.key?(:after)
184
+ app.middleware.insert_after(middleware_position[:after], Skylight::Middleware, config: config)
185
+ elsif middleware_position.key?(:before)
186
+ app.middleware.insert_before(middleware_position[:before], Skylight::Middleware, config: config)
187
+ else
188
+ raise "The middleware position you have set is invalid. Please be sure " \
189
+ "`config.skylight.middleware_position` is set up correctly."
190
+ end
191
+ end
192
+
193
+ def set_middleware_position(app, config)
194
+ if middleware_position.is_a?(Integer)
195
+ app.middleware.insert middleware_position, Skylight::Middleware, config: config
196
+ elsif middleware_position.is_a?(Hash) && middleware_position.keys.count == 1
197
+ insert_middleware(app, config)
198
+ elsif middleware_position.nil?
199
+ app.middleware.insert 0, Skylight::Middleware, config: config
200
+ else
201
+ raise "The middleware position you have set is invalid. Please be sure " \
202
+ "`config.skylight.middleware_position` is set up correctly."
203
+ end
204
+ end
205
+
206
+ def sk_rails_config(target = self)
207
+ target.config.skylight
208
+ end
65
209
  end
66
210
  end