skylight 4.3.2 → 5.0.0.beta5

Sign up to get free protection for your applications and to get access to all the features.
Files changed (113) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +36 -3
  3. data/CONTRIBUTING.md +1 -7
  4. data/ext/extconf.rb +6 -5
  5. data/ext/libskylight.yml +7 -6
  6. data/ext/skylight_native.c +22 -99
  7. data/lib/skylight.rb +211 -14
  8. data/lib/skylight/api.rb +10 -3
  9. data/lib/skylight/cli.rb +4 -3
  10. data/lib/skylight/cli/doctor.rb +13 -14
  11. data/lib/skylight/cli/merger.rb +6 -4
  12. data/lib/skylight/config.rb +605 -127
  13. data/lib/skylight/deprecation.rb +17 -0
  14. data/lib/skylight/errors.rb +21 -6
  15. data/lib/skylight/extensions.rb +107 -0
  16. data/lib/skylight/extensions/source_location.rb +291 -0
  17. data/lib/skylight/formatters/http.rb +20 -0
  18. data/lib/skylight/gc.rb +109 -0
  19. data/lib/skylight/helpers.rb +52 -27
  20. data/lib/skylight/instrumenter.rb +326 -15
  21. data/lib/skylight/middleware.rb +138 -1
  22. data/lib/skylight/native.rb +52 -2
  23. data/lib/skylight/native_ext_fetcher.rb +4 -3
  24. data/lib/skylight/normalizers.rb +153 -0
  25. data/lib/skylight/normalizers/action_controller/process_action.rb +69 -0
  26. data/lib/skylight/normalizers/action_controller/send_file.rb +50 -0
  27. data/lib/skylight/normalizers/action_dispatch/process_middleware.rb +22 -0
  28. data/lib/skylight/normalizers/action_dispatch/route_set.rb +27 -0
  29. data/lib/skylight/normalizers/action_view/render_collection.rb +24 -0
  30. data/lib/skylight/normalizers/action_view/render_layout.rb +25 -0
  31. data/lib/skylight/normalizers/action_view/render_partial.rb +23 -0
  32. data/lib/skylight/normalizers/action_view/render_template.rb +23 -0
  33. data/lib/skylight/normalizers/active_job/perform.rb +86 -0
  34. data/lib/skylight/normalizers/active_model_serializers/render.rb +28 -0
  35. data/lib/skylight/normalizers/active_record/instantiation.rb +16 -0
  36. data/lib/skylight/normalizers/active_record/sql.rb +12 -0
  37. data/lib/skylight/normalizers/active_storage.rb +30 -0
  38. data/lib/skylight/normalizers/active_support/cache.rb +22 -0
  39. data/lib/skylight/normalizers/active_support/cache_clear.rb +16 -0
  40. data/lib/skylight/normalizers/active_support/cache_decrement.rb +16 -0
  41. data/lib/skylight/normalizers/active_support/cache_delete.rb +16 -0
  42. data/lib/skylight/normalizers/active_support/cache_exist.rb +16 -0
  43. data/lib/skylight/normalizers/active_support/cache_fetch_hit.rb +16 -0
  44. data/lib/skylight/normalizers/active_support/cache_generate.rb +16 -0
  45. data/lib/skylight/normalizers/active_support/cache_increment.rb +16 -0
  46. data/lib/skylight/normalizers/active_support/cache_read.rb +16 -0
  47. data/lib/skylight/normalizers/active_support/cache_read_multi.rb +16 -0
  48. data/lib/skylight/normalizers/active_support/cache_write.rb +16 -0
  49. data/lib/skylight/normalizers/coach/handler_finish.rb +46 -0
  50. data/lib/skylight/normalizers/coach/middleware_finish.rb +33 -0
  51. data/lib/skylight/normalizers/couch_potato/query.rb +20 -0
  52. data/lib/skylight/normalizers/data_mapper/sql.rb +12 -0
  53. data/lib/skylight/normalizers/default.rb +32 -0
  54. data/lib/skylight/normalizers/elasticsearch/request.rb +20 -0
  55. data/lib/skylight/normalizers/faraday/request.rb +40 -0
  56. data/lib/skylight/normalizers/grape/endpoint.rb +34 -0
  57. data/lib/skylight/normalizers/grape/endpoint_render.rb +25 -0
  58. data/lib/skylight/normalizers/grape/endpoint_run.rb +41 -0
  59. data/lib/skylight/normalizers/grape/endpoint_run_filters.rb +22 -0
  60. data/lib/skylight/normalizers/grape/format_response.rb +20 -0
  61. data/lib/skylight/normalizers/graphiti/render.rb +22 -0
  62. data/lib/skylight/normalizers/graphiti/resolve.rb +31 -0
  63. data/lib/skylight/normalizers/graphql/base.rb +132 -0
  64. data/lib/skylight/normalizers/render.rb +81 -0
  65. data/lib/skylight/normalizers/sequel/sql.rb +12 -0
  66. data/lib/skylight/normalizers/shrine.rb +34 -0
  67. data/lib/skylight/normalizers/sql.rb +45 -0
  68. data/lib/skylight/probes.rb +181 -0
  69. data/lib/skylight/probes/action_controller.rb +48 -0
  70. data/lib/skylight/probes/action_dispatch.rb +2 -0
  71. data/lib/skylight/probes/action_dispatch/request_id.rb +29 -0
  72. data/lib/skylight/probes/action_dispatch/routing/route_set.rb +28 -0
  73. data/lib/skylight/probes/action_view.rb +43 -0
  74. data/lib/skylight/probes/active_job.rb +27 -0
  75. data/lib/skylight/probes/active_job_enqueue.rb +41 -0
  76. data/lib/skylight/probes/active_model_serializers.rb +50 -0
  77. data/lib/skylight/probes/delayed_job.rb +149 -0
  78. data/lib/skylight/probes/elasticsearch.rb +38 -0
  79. data/lib/skylight/probes/excon.rb +25 -0
  80. data/lib/skylight/probes/excon/middleware.rb +66 -0
  81. data/lib/skylight/probes/faraday.rb +23 -0
  82. data/lib/skylight/probes/graphql.rb +43 -0
  83. data/lib/skylight/probes/httpclient.rb +44 -0
  84. data/lib/skylight/probes/middleware.rb +126 -0
  85. data/lib/skylight/probes/mongo.rb +164 -0
  86. data/lib/skylight/probes/mongoid.rb +13 -0
  87. data/lib/skylight/probes/net_http.rb +54 -0
  88. data/lib/skylight/probes/redis.rb +63 -0
  89. data/lib/skylight/probes/sequel.rb +33 -0
  90. data/lib/skylight/probes/sinatra.rb +63 -0
  91. data/lib/skylight/probes/sinatra_add_middleware.rb +10 -10
  92. data/lib/skylight/probes/tilt.rb +27 -0
  93. data/lib/skylight/railtie.rb +162 -18
  94. data/lib/skylight/sidekiq.rb +48 -0
  95. data/lib/skylight/subscriber.rb +110 -0
  96. data/lib/skylight/test.rb +146 -0
  97. data/lib/skylight/trace.rb +307 -10
  98. data/lib/skylight/user_config.rb +61 -0
  99. data/lib/skylight/util.rb +12 -0
  100. data/lib/skylight/util/allocation_free.rb +26 -0
  101. data/lib/skylight/util/clock.rb +56 -0
  102. data/lib/skylight/util/component.rb +5 -2
  103. data/lib/skylight/util/deploy.rb +7 -10
  104. data/lib/skylight/util/gzip.rb +20 -0
  105. data/lib/skylight/util/http.rb +4 -10
  106. data/lib/skylight/util/instrumenter_method.rb +26 -0
  107. data/lib/skylight/util/logging.rb +138 -0
  108. data/lib/skylight/util/lru_cache.rb +40 -0
  109. data/lib/skylight/util/platform.rb +1 -1
  110. data/lib/skylight/vendor/cli/thor/rake_compat.rb +1 -1
  111. data/lib/skylight/version.rb +5 -1
  112. data/lib/skylight/vm/gc.rb +68 -0
  113. metadata +114 -15
@@ -1,4 +1,141 @@
1
+ require "securerandom"
2
+
1
3
  module Skylight
2
- class Middleware < Core::Middleware
4
+ # @api private
5
+ class Middleware
6
+ class BodyProxy
7
+ def initialize(body, &block)
8
+ @body = body
9
+ @block = block
10
+ @closed = false
11
+ end
12
+
13
+ def respond_to_missing?(name, include_all = false)
14
+ return false if name.to_s !~ /^to_ary$/
15
+
16
+ @body.respond_to?(name, include_all)
17
+ end
18
+
19
+ def close
20
+ return if @closed
21
+
22
+ @closed = true
23
+ begin
24
+ @body.close if @body.respond_to? :close
25
+ ensure
26
+ @block.call
27
+ end
28
+ end
29
+
30
+ def closed?
31
+ @closed
32
+ end
33
+
34
+ # N.B. This method is a special case to address the bug described by
35
+ # https://github.com/rack/rack/issues/434.
36
+ # We are applying this special case for #each only. Future bugs of this
37
+ # class will be handled by requesting users to patch their ruby
38
+ # implementation, to save adding too many methods in this class.
39
+ def each(*args, &block)
40
+ @body.each(*args, &block)
41
+ end
42
+
43
+ def method_missing(*args, &block)
44
+ super if args.first.to_s =~ /^to_ary$/
45
+ @body.__send__(*args, &block)
46
+ end
47
+ end
48
+
49
+ def self.with_after_close(resp, debug_identifier: "unknown", &block)
50
+ unless resp.respond_to?(:to_ary)
51
+ if resp.respond_to?(:to_a)
52
+ Skylight.warn("Rack response from \"#{debug_identifier}\" cannot be implicitly converted to an array. " \
53
+ "This is in violation of the Rack SPEC and will raise an error in future versions.")
54
+ resp = resp.to_a
55
+ else
56
+ Skylight.error("Rack response from \"#{debug_identifier}\" cannot be converted to an array. This is in " \
57
+ "violation of the Rack SPEC and may cause problems with Skylight operation.")
58
+ return resp
59
+ end
60
+ end
61
+
62
+ status, headers, body = resp
63
+ [status, headers, BodyProxy.new(body, &block)]
64
+ end
65
+
66
+ include Skylight::Util::Logging
67
+
68
+ # For Util::Logging
69
+ attr_reader :config
70
+
71
+ def initialize(app, opts = {})
72
+ @app = app
73
+ @config = opts[:config]
74
+ end
75
+
76
+ def call(env)
77
+ set_request_id(env)
78
+
79
+ if Skylight.tracing?
80
+ error "Already instrumenting. Make sure the Skylight Rack Middleware hasn't been added more than once."
81
+ end
82
+
83
+ if env["REQUEST_METHOD"] == "HEAD"
84
+ t { "middleware skipping HEAD" }
85
+ @app.call(env)
86
+ else
87
+ begin
88
+ t { "middleware beginning trace" }
89
+ trace = Skylight.trace(endpoint_name(env), "app.rack.request", nil, meta: endpoint_meta(env), component: :web)
90
+ t { "middleware began trace=#{trace ? trace.uuid : nil}" }
91
+
92
+ resp = @app.call(env)
93
+
94
+ if trace
95
+ Middleware.with_after_close(resp, debug_identifier: "Rack App: #{@app.class}") { trace.submit }
96
+ else
97
+ resp
98
+ end
99
+ rescue Exception => e
100
+ t { "middleware exception: #{e}\n#{e.backtrace.join("\n")}" }
101
+ trace&.submit
102
+ raise
103
+ end
104
+ end
105
+ end
106
+
107
+ private
108
+
109
+ def log_context
110
+ # Don't cache this, it will change
111
+ { request_id: @current_request_id, inst: Skylight.instrumenter&.uuid }
112
+ end
113
+
114
+ # Allow for overwriting
115
+ def endpoint_name(_env)
116
+ "Rack"
117
+ end
118
+
119
+ def endpoint_meta(_env)
120
+ { source_location: Trace::SYNTHETIC }
121
+ end
122
+
123
+ # Request ID code based on ActionDispatch::RequestId
124
+ def set_request_id(env)
125
+ existing_request_id = env["action_dispatch.request_id"] || env["HTTP_X_REQUEST_ID"]
126
+ @current_request_id = env["skylight.request_id"] = make_request_id(existing_request_id)
127
+ end
128
+
129
+ def make_request_id(request_id)
130
+ if request_id && !request_id.empty?
131
+ request_id.gsub(/[^\w\-]/, "".freeze)[0...255]
132
+ else
133
+ internal_request_id
134
+ end
135
+ end
136
+
137
+ def internal_request_id
138
+ SecureRandom.uuid
139
+ end
3
140
  end
4
141
  end
@@ -1,6 +1,55 @@
1
1
  require "skylight/util/platform"
2
2
 
3
3
  module Skylight
4
+ # Some methods exepected to be defined by the native code (OUTDATED)
5
+ #
6
+ # * Skylight::Util::Clock#native_hrtime
7
+ # - returns current time in nanoseconds
8
+ # * Skylight::Trace#native_new(start, uuid, endpoint)
9
+ # - start is milliseconds
10
+ # - uuid is currently unused
11
+ # - endpoint is the endpoint name
12
+ # - returns an instance of Trace
13
+ # * Skylight::Trace#native_get_started_at
14
+ # - returns the start time
15
+ # * Skylight::Trace#native_get_endpoint
16
+ # - returns the endpoint name
17
+ # * Skylight::Trace#native_set_endpoint(endpoint)
18
+ # - returns nil
19
+ # * Skylight::Trace#native_get_uuid
20
+ # - returns the uuid
21
+ # * Skylight::Trace#native_start_span(time, category)
22
+ # - time is milliseconds
23
+ # - category is a string
24
+ # - returns a numeric span id
25
+ # * Skylight::Trace#native_stop_span(span, time)
26
+ # - span is the span id
27
+ # - time is milliseconds
28
+ # - returns nil
29
+ # * Skylight::Trace#native_span_set_title(span, title)
30
+ # - span is the span id
31
+ # - title is a string
32
+ # - returns nil
33
+ # * Skylight::Trace#native_span_set_description(span, desc)
34
+ # - span is the span id
35
+ # - desc is a string
36
+ # - returns nil
37
+ # * Skylight::Instrumenter#native_new(env)
38
+ # - env is the config converted to a flattened array of ENV style values
39
+ # e.g. `["SKYLIGHT_AUTHENTICATION", "abc123", ...]
40
+ # - returns a new Instrumenter instance
41
+ # * Skylight::Instrumenter#native_start()
42
+ # - returns a truthy value if successful
43
+ # * Skylight::Instrumenter#native_stop()
44
+ # - returns nil
45
+ # * Skylight::Instrumenter#native_submit_trace(trace)
46
+ # - trace is a Trace instance
47
+ # - returns nil
48
+ # * Skylight::Instrumenter#native_track_desc(endpoint, description)
49
+ # - endpoint is a string
50
+ # - description is a string
51
+ # - returns truthy unless uniqueness cap exceeded
52
+
4
53
  # @api private
5
54
  # Whether or not the native extension is present
6
55
  @has_native_ext = false
@@ -40,7 +89,8 @@ module Skylight
40
89
  end
41
90
 
42
91
  if Skylight.native?
43
- Skylight::Core::Util::Clock.use_native!
92
+ require "skylight/util/clock"
93
+ Util::Clock.use_native!
44
94
  else
45
95
  class Instrumenter
46
96
  def self.native_new(*_args)
@@ -51,7 +101,7 @@ module Skylight
51
101
 
52
102
  # @api private
53
103
  def self.check_install_errors(config)
54
- # Note: An unsupported arch doesn't count as an error.
104
+ # NOTE: An unsupported arch doesn't count as an error.
55
105
  install_log = File.expand_path("../../ext/install.log", __dir__)
56
106
 
57
107
  if File.exist?(install_log) && File.read(install_log) =~ /ERROR/
@@ -23,7 +23,7 @@ module Skylight
23
23
  # @param opts [Hash]
24
24
  def self.fetch(**args)
25
25
  args[:source] ||= BASE_URL
26
- args[:logger] ||= Logger.new(STDOUT)
26
+ args[:logger] ||= Logger.new($stdout)
27
27
  new(**args).fetch
28
28
  end
29
29
 
@@ -35,7 +35,7 @@ module Skylight
35
35
  # @param required [Boolean] whether the download is required to be successful
36
36
  # @param platform
37
37
  # @param log [Logger]
38
- def initialize(source:, target:, version:, checksum:, arch:, required: false, platform: nil, logger:)
38
+ def initialize(source:, target:, version:, checksum:, arch:, logger:, required: false, platform: nil)
39
39
  raise "source required" unless source
40
40
  raise "target required" unless target
41
41
  raise "checksum required" unless checksum
@@ -112,7 +112,8 @@ module Skylight
112
112
  rescue => e
113
113
  remaining_attempts -= 1
114
114
 
115
- error "failed to fetch native extension; uri=#{uri}; msg=#{e.message}; remaining-attempts=#{remaining_attempts}", e
115
+ error "failed to fetch native extension; uri=#{uri}; msg=#{e.message}; " \
116
+ "remaining-attempts=#{remaining_attempts}", e
116
117
 
117
118
  if remaining_attempts > 0
118
119
  sleep 2
@@ -0,0 +1,153 @@
1
+ module Skylight
2
+ # @api private
3
+ # Convert AS::N events to Skylight events
4
+ module Normalizers
5
+ def self.registry
6
+ @registry ||= {}
7
+ end
8
+
9
+ def self.register(name, klass, opts = {})
10
+ enabled = opts[:enabled] != false
11
+ registry[name] = [klass, enabled]
12
+ end
13
+
14
+ def self.unregister(name)
15
+ @registry.delete(name)
16
+ end
17
+
18
+ def self.enable(*names, enabled: true)
19
+ names.each do |name|
20
+ matches = registry.select { |n, _| n =~ /(^|\.)#{name}$/ }
21
+ raise ArgumentError, "no normalizers match #{name}" if matches.empty?
22
+
23
+ matches.each_value { |v| v[1] = enabled }
24
+ end
25
+ end
26
+
27
+ def self.disable(*names)
28
+ enable(*names, enabled: false)
29
+ end
30
+
31
+ def self.build(config)
32
+ normalizers = {}
33
+
34
+ registry.each do |key, (klass, enabled)|
35
+ next unless enabled
36
+
37
+ unless klass.method_defined?(:normalize)
38
+ # TODO: Warn
39
+ next
40
+ end
41
+
42
+ normalizers[key] = klass.new(config)
43
+ end
44
+
45
+ Container.new(normalizers)
46
+ end
47
+
48
+ class Normalizer
49
+ def self.register(name, opts = {})
50
+ Normalizers.register(name, self, opts)
51
+ end
52
+
53
+ attr_reader :config
54
+
55
+ include Util::Logging
56
+
57
+ def initialize(config)
58
+ @config = config
59
+ setup if respond_to?(:setup)
60
+ end
61
+
62
+ def normalize(_trace, _name, _payload)
63
+ :skip
64
+ end
65
+
66
+ def normalize_with_meta(trace, name, payload)
67
+ # If we have a normal response but no meta, add it
68
+ cat, title, desc, meta = ret = normalize(trace, name, payload)
69
+ return cat if cat == :skip
70
+
71
+ meta ||= {}
72
+ cache_key = ret.hash
73
+ process_meta(trace, name, payload, meta, cache_key: cache_key)
74
+
75
+ [cat, title, desc, meta]
76
+ end
77
+
78
+ def normalize_after(trace, span, name, payload); end
79
+
80
+ private
81
+
82
+ def process_meta(trace, _name, payload, meta, cache_key: nil)
83
+ trace.instrumenter.extensions.process_normalizer_meta(
84
+ payload,
85
+ meta,
86
+ cache_key: cache_key,
87
+ **process_meta_options(payload)
88
+ )
89
+ end
90
+
91
+ def process_meta_options(_payload)
92
+ {}
93
+ end
94
+ end
95
+
96
+ require "skylight/normalizers/default"
97
+ DEFAULT = Default.new
98
+
99
+ class Container
100
+ def initialize(normalizers)
101
+ @normalizers = normalizers
102
+ end
103
+
104
+ def keys
105
+ @normalizers.keys
106
+ end
107
+
108
+ def normalize(trace, name, payload)
109
+ normalizer_for(name).normalize_with_meta(trace, name, payload)
110
+ end
111
+
112
+ def normalize_after(trace, span, name, payload)
113
+ normalizer_for(name).normalize_after(trace, span, name, payload)
114
+ end
115
+
116
+ def normalizer_for(name)
117
+ # We never expect to hit the default case since we only register listeners
118
+ # for items that we know have normalizers. For now, though, we'll play it
119
+ # safe and provide a fallback.
120
+ @normalizers.fetch(name, DEFAULT)
121
+ end
122
+ end
123
+
124
+ %w[ action_controller/process_action
125
+ action_controller/send_file
126
+ action_dispatch/process_middleware
127
+ action_dispatch/route_set
128
+ action_view/render_collection
129
+ action_view/render_partial
130
+ action_view/render_template
131
+ action_view/render_layout
132
+ active_job/perform
133
+ active_model_serializers/render
134
+ active_record/instantiation
135
+ active_record/sql
136
+ active_storage
137
+ active_support/cache
138
+ coach/handler_finish
139
+ coach/middleware_finish
140
+ couch_potato/query
141
+ data_mapper/sql
142
+ elasticsearch/request
143
+ faraday/request
144
+ grape/endpoint
145
+ graphiti/resolve
146
+ graphiti/render
147
+ graphql/base
148
+ sequel/sql
149
+ shrine].each do |file|
150
+ require "skylight/normalizers/#{file}"
151
+ end
152
+ end
153
+ end
@@ -0,0 +1,69 @@
1
+ module Skylight
2
+ module Normalizers
3
+ module ActionController
4
+ # Normalizer for processing a Rails controller action
5
+ class ProcessAction < Normalizer
6
+ register "process_action.action_controller"
7
+
8
+ CAT = "app.controller.request".freeze
9
+
10
+ # Payload Keys: controller, action, params, format, method, path
11
+ # Additional keys available in `normalize_after`: status, view_runtime
12
+ # Along with ones added by probe: variant
13
+
14
+ # @param trace [Skylight::Messages::Trace::Builder]
15
+ # @param name [String] ignored, only present to match API
16
+ # @param payload [Hash]
17
+ # @option payload [String] :controller Controller name
18
+ # @option payload [String] :action Action name
19
+ # @return [Array]
20
+ def normalize(trace, _name, payload)
21
+ trace.endpoint = controller_action(payload)
22
+ [CAT, trace.endpoint, nil]
23
+ end
24
+
25
+ def normalize_after(trace, _span, _name, payload)
26
+ return unless config.enable_segments?
27
+
28
+ if (segment = segment_from_payload(payload))
29
+ trace.segment = segment
30
+ end
31
+ end
32
+
33
+ private
34
+
35
+ def controller_action(payload)
36
+ "#{payload[:controller]}##{payload[:action]}"
37
+ end
38
+
39
+ def process_meta_options(payload)
40
+ # provide hints to override default source_location behavior
41
+ super.merge(source_location_hint: [:instance_method, payload[:controller], payload[:action]])
42
+ end
43
+
44
+ def segment_from_payload(payload)
45
+ # Show 'error' if there's an unhandled exception or if the status is 4xx or 5xx
46
+ return "error" if payload[:exception] || payload[:exception_object]
47
+
48
+ segment_from_status(payload[:status]) ||
49
+ if payload[:sk_rendered_format]
50
+ # We only show the variant if we actually have a format
51
+ # We won't have a sk_rendered_format if it's a `head` outside of a `respond_to` block.
52
+ [payload[:sk_rendered_format], payload[:sk_variant]].compact.flatten.join("+")
53
+ end
54
+ end
55
+
56
+ def segment_from_status(status)
57
+ case status
58
+ when 304
59
+ "not modified"
60
+ when (300..399)
61
+ "redirect"
62
+ when (400..599)
63
+ "error"
64
+ end
65
+ end
66
+ end
67
+ end
68
+ end
69
+ end