skylight 4.2.1 → 5.0.0.beta

Sign up to get free protection for your applications and to get access to all the features.
Files changed (112) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +27 -0
  3. data/CONTRIBUTING.md +1 -7
  4. data/ext/extconf.rb +4 -3
  5. data/ext/libskylight.yml +5 -6
  6. data/ext/skylight_native.c +24 -100
  7. data/lib/skylight.rb +204 -14
  8. data/lib/skylight/api.rb +7 -3
  9. data/lib/skylight/cli.rb +4 -3
  10. data/lib/skylight/cli/doctor.rb +3 -2
  11. data/lib/skylight/cli/merger.rb +6 -4
  12. data/lib/skylight/config.rb +603 -126
  13. data/lib/skylight/deprecation.rb +15 -0
  14. data/lib/skylight/errors.rb +17 -2
  15. data/lib/skylight/extensions.rb +99 -0
  16. data/lib/skylight/extensions/source_location.rb +249 -0
  17. data/lib/skylight/fanout.rb +0 -0
  18. data/lib/skylight/formatters/http.rb +19 -0
  19. data/lib/skylight/gc.rb +109 -0
  20. data/lib/skylight/helpers.rb +18 -2
  21. data/lib/skylight/instrumenter.rb +325 -15
  22. data/lib/skylight/middleware.rb +138 -1
  23. data/lib/skylight/native.rb +51 -1
  24. data/lib/skylight/native_ext_fetcher.rb +2 -1
  25. data/lib/skylight/normalizers.rb +151 -0
  26. data/lib/skylight/normalizers/action_controller/process_action.rb +69 -0
  27. data/lib/skylight/normalizers/action_controller/send_file.rb +50 -0
  28. data/lib/skylight/normalizers/action_dispatch/process_middleware.rb +22 -0
  29. data/lib/skylight/normalizers/action_dispatch/route_set.rb +27 -0
  30. data/lib/skylight/normalizers/action_view/render_collection.rb +24 -0
  31. data/lib/skylight/normalizers/action_view/render_layout.rb +25 -0
  32. data/lib/skylight/normalizers/action_view/render_partial.rb +23 -0
  33. data/lib/skylight/normalizers/action_view/render_template.rb +23 -0
  34. data/lib/skylight/normalizers/active_job/perform.rb +81 -0
  35. data/lib/skylight/normalizers/active_model_serializers/render.rb +28 -0
  36. data/lib/skylight/normalizers/active_record/instantiation.rb +16 -0
  37. data/lib/skylight/normalizers/active_record/sql.rb +12 -0
  38. data/lib/skylight/normalizers/active_storage.rb +30 -0
  39. data/lib/skylight/normalizers/active_support/cache.rb +22 -0
  40. data/lib/skylight/normalizers/active_support/cache_clear.rb +16 -0
  41. data/lib/skylight/normalizers/active_support/cache_decrement.rb +16 -0
  42. data/lib/skylight/normalizers/active_support/cache_delete.rb +16 -0
  43. data/lib/skylight/normalizers/active_support/cache_exist.rb +16 -0
  44. data/lib/skylight/normalizers/active_support/cache_fetch_hit.rb +16 -0
  45. data/lib/skylight/normalizers/active_support/cache_generate.rb +16 -0
  46. data/lib/skylight/normalizers/active_support/cache_increment.rb +16 -0
  47. data/lib/skylight/normalizers/active_support/cache_read.rb +16 -0
  48. data/lib/skylight/normalizers/active_support/cache_read_multi.rb +16 -0
  49. data/lib/skylight/normalizers/active_support/cache_write.rb +16 -0
  50. data/lib/skylight/normalizers/coach/handler_finish.rb +46 -0
  51. data/lib/skylight/normalizers/coach/middleware_finish.rb +33 -0
  52. data/lib/skylight/normalizers/couch_potato/query.rb +20 -0
  53. data/lib/skylight/normalizers/data_mapper/sql.rb +12 -0
  54. data/lib/skylight/normalizers/default.rb +32 -0
  55. data/lib/skylight/normalizers/elasticsearch/request.rb +20 -0
  56. data/lib/skylight/normalizers/faraday/request.rb +40 -0
  57. data/lib/skylight/normalizers/grape/endpoint.rb +34 -0
  58. data/lib/skylight/normalizers/grape/endpoint_render.rb +25 -0
  59. data/lib/skylight/normalizers/grape/endpoint_run.rb +41 -0
  60. data/lib/skylight/normalizers/grape/endpoint_run_filters.rb +22 -0
  61. data/lib/skylight/normalizers/grape/format_response.rb +20 -0
  62. data/lib/skylight/normalizers/graphiti/render.rb +22 -0
  63. data/lib/skylight/normalizers/graphiti/resolve.rb +31 -0
  64. data/lib/skylight/normalizers/graphql/base.rb +131 -0
  65. data/lib/skylight/normalizers/render.rb +81 -0
  66. data/lib/skylight/normalizers/sequel/sql.rb +12 -0
  67. data/lib/skylight/normalizers/sql.rb +44 -0
  68. data/lib/skylight/probes.rb +153 -0
  69. data/lib/skylight/probes/action_controller.rb +48 -0
  70. data/lib/skylight/probes/action_dispatch.rb +2 -0
  71. data/lib/skylight/probes/action_dispatch/request_id.rb +29 -0
  72. data/lib/skylight/probes/action_dispatch/routing/route_set.rb +28 -0
  73. data/lib/skylight/probes/action_view.rb +43 -0
  74. data/lib/skylight/probes/active_job.rb +29 -0
  75. data/lib/skylight/probes/active_job_enqueue.rb +37 -0
  76. data/lib/skylight/probes/active_model_serializers.rb +54 -0
  77. data/lib/skylight/probes/delayed_job.rb +62 -0
  78. data/lib/skylight/probes/elasticsearch.rb +38 -0
  79. data/lib/skylight/probes/excon.rb +25 -0
  80. data/lib/skylight/probes/excon/middleware.rb +66 -0
  81. data/lib/skylight/probes/faraday.rb +23 -0
  82. data/lib/skylight/probes/graphql.rb +43 -0
  83. data/lib/skylight/probes/httpclient.rb +44 -0
  84. data/lib/skylight/probes/middleware.rb +125 -0
  85. data/lib/skylight/probes/mongo.rb +163 -0
  86. data/lib/skylight/probes/mongoid.rb +13 -0
  87. data/lib/skylight/probes/net_http.rb +55 -0
  88. data/lib/skylight/probes/redis.rb +60 -0
  89. data/lib/skylight/probes/sequel.rb +33 -0
  90. data/lib/skylight/probes/sinatra.rb +63 -0
  91. data/lib/skylight/probes/sinatra_add_middleware.rb +10 -10
  92. data/lib/skylight/probes/tilt.rb +27 -0
  93. data/lib/skylight/railtie.rb +162 -18
  94. data/lib/skylight/sidekiq.rb +43 -0
  95. data/lib/skylight/subscriber.rb +110 -0
  96. data/lib/skylight/test.rb +146 -0
  97. data/lib/skylight/trace.rb +301 -10
  98. data/lib/skylight/user_config.rb +61 -0
  99. data/lib/skylight/util.rb +12 -0
  100. data/lib/skylight/util/allocation_free.rb +26 -0
  101. data/lib/skylight/util/clock.rb +56 -0
  102. data/lib/skylight/util/component.rb +5 -2
  103. data/lib/skylight/util/deploy.rb +4 -4
  104. data/lib/skylight/util/gzip.rb +20 -0
  105. data/lib/skylight/util/http.rb +4 -10
  106. data/lib/skylight/util/instrumenter_method.rb +26 -0
  107. data/lib/skylight/util/logging.rb +138 -0
  108. data/lib/skylight/util/lru_cache.rb +42 -0
  109. data/lib/skylight/vendor/cli/thor/rake_compat.rb +1 -1
  110. data/lib/skylight/version.rb +5 -1
  111. data/lib/skylight/vm/gc.rb +68 -0
  112. metadata +116 -17
@@ -0,0 +1,81 @@
1
+ module Skylight
2
+ module Normalizers
3
+ # Base Normalizer for Rails rendering
4
+ class RenderNormalizer < Normalizer
5
+ include Skylight::Util::AllocationFree
6
+
7
+ def setup
8
+ @paths = []
9
+
10
+ Gem.path.each do |path|
11
+ @paths << "#{path}/bundler/gems".freeze
12
+ @paths << "#{path}/gems".freeze
13
+ @paths << path
14
+ end
15
+
16
+ @paths.concat(Array(config["normalizers.render.view_paths"]))
17
+ end
18
+
19
+ # Generic normalizer for renders
20
+ # @param category [String]
21
+ # @param payload [Hash]
22
+ # @option payload [String] :identifier
23
+ # @return [Array]
24
+ def normalize_render(category, payload)
25
+ if (path = payload[:identifier])
26
+ title = relative_path(path)
27
+ end
28
+
29
+ [category, title, nil]
30
+ end
31
+
32
+ def relative_path(path)
33
+ return path if relative_path?(path)
34
+
35
+ if (root = array_find(@paths) { |p| path.start_with?(p) })
36
+ start = root.size
37
+ start += 1 if path.getbyte(start) == SEPARATOR_BYTE
38
+
39
+ path[start, path.size].sub(
40
+ # Matches a Gem Version or 12-digit hex (sha)
41
+ # that is preceeded by a `-` and followed by `/`
42
+ # Also matches 'app/views/' if it exists
43
+ %r{-(?:#{Gem::Version::VERSION_PATTERN}|[0-9a-f]{12})\/(?:app\/views\/)*},
44
+ ": ".freeze
45
+ )
46
+ else
47
+ "Absolute Path".freeze
48
+ end
49
+ end
50
+
51
+ private
52
+
53
+ def relative_path?(path)
54
+ !absolute_path?(path)
55
+ end
56
+
57
+ SEPARATOR_BYTE = File::SEPARATOR.ord
58
+
59
+ if File.const_defined?(:NULL) ? File::NULL == "NUL" : RbConfig::CONFIG["host_os"] =~ /mingw|mswin32/
60
+ # This is a DOSish environment
61
+ ALT_SEPARATOR_BYTE = File::ALT_SEPARATOR&.ord
62
+ COLON_BYTE = ":".ord
63
+ SEPARATOR_BYTES = [SEPARATOR_BYTE, ALT_SEPARATOR_BYTE].freeze
64
+
65
+ def absolute_path?(path)
66
+ if alpha?(path.getbyte(0)) && path.getbyte(1) == COLON_BYTE
67
+ SEPARATOR_BYTES.include?(path.getbyte(2))
68
+ end
69
+ end
70
+
71
+ def alpha?(byte)
72
+ (byte >= 65 && byte <= 90) || (byte >= 97 && byte <= 122)
73
+ end
74
+ else
75
+ def absolute_path?(path)
76
+ path.getbyte(0) == SEPARATOR_BYTE
77
+ end
78
+ end
79
+ end
80
+ end
81
+ end
@@ -0,0 +1,12 @@
1
+ require "skylight/normalizers/sql"
2
+
3
+ module Skylight
4
+ module Normalizers
5
+ module Sequel
6
+ # Normalizer for SQL requests
7
+ class SQL < Normalizers::SQL
8
+ register "sql.sequel"
9
+ end
10
+ end
11
+ end
12
+ end
@@ -0,0 +1,44 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+
5
+ module Skylight
6
+ module Normalizers
7
+ # Normalizer for SQL requests
8
+ class SQL < Normalizer
9
+ CAT = "db.sql.query"
10
+
11
+ # @param trace [Skylight::Messages::Trace::Builder] ignored, only present to match API
12
+ # @param name [String] ignored, only present to match API
13
+ # @param payload [Hash]
14
+ # @option payload [String] [:name] The SQL operation
15
+ # @option payload [Hash] [:binds] The bound parameters
16
+ # @return [Array]
17
+ def normalize(trace, name, payload)
18
+ case payload[:name]
19
+ when "SCHEMA", "CACHE"
20
+ return :skip
21
+ else
22
+ name = CAT
23
+ title = payload[:name] || "SQL"
24
+ end
25
+
26
+ # We can only handle UTF-8 encoded strings.
27
+ # (Construction method here avoids extra allocations)
28
+ sql = String.new.concat("<sk-sql>", payload[:sql], "</sk-sql>").force_encoding(Encoding::UTF_8)
29
+
30
+ unless sql.valid_encoding?
31
+ if config[:log_sql_parse_errors]
32
+ config.logger.error "[#{Skylight::SqlLexError.formatted_code}] Unable to extract binds from non-UTF-8 query. " \
33
+ "encoding=#{payload[:sql].encoding.name} " \
34
+ "sql=#{payload[:sql].inspect} "
35
+ end
36
+
37
+ sql = nil
38
+ end
39
+
40
+ [name, title, sql]
41
+ end
42
+ end
43
+ end
44
+ end
@@ -0,0 +1,153 @@
1
+ require "pathname"
2
+ require "active_support/inflector"
3
+
4
+ module Skylight
5
+ # @api private
6
+ module Probes
7
+ class ProbeRegistration
8
+ attr_reader :name, :const_name, :require_paths, :probe
9
+
10
+ def initialize(name, const_name, require_paths, probe)
11
+ @name = name
12
+ @const_name = const_name
13
+ @require_paths = Array(require_paths)
14
+ @probe = probe
15
+ end
16
+
17
+ def install
18
+ probe.install
19
+ end
20
+
21
+ def constant_available?
22
+ Skylight::Probes.constant_available?(const_name)
23
+ end
24
+ end
25
+
26
+ class << self
27
+ def constant_available?(const_name)
28
+ ::ActiveSupport::Inflector.safe_constantize(const_name).present?
29
+ end
30
+
31
+ def install!
32
+ pending = registered.values - installed.values
33
+
34
+ pending.each do |registration|
35
+ if registration.constant_available?
36
+ install_probe(registration)
37
+ else
38
+ register_require_hook(registration)
39
+ end
40
+ end
41
+ end
42
+
43
+ def install_probe(registration)
44
+ return if installed.key?(registration.name)
45
+
46
+ installed[registration.name] = registration
47
+ registration.install
48
+ end
49
+
50
+ def add_path(path)
51
+ root = Pathname.new(path)
52
+ Pathname.glob(root.join("./**/*.rb")).each do |f|
53
+ name = f.relative_path_from(root).sub_ext("").to_s
54
+ if available.key?(name)
55
+ raise "duplicate probe name: #{name}; original=#{available[name]}; new=#{f}"
56
+ end
57
+
58
+ available[name] = f
59
+ end
60
+ end
61
+
62
+ def available
63
+ @available ||= {}
64
+ end
65
+
66
+ def probe(*probes)
67
+ unknown = probes.map(&:to_s) - available.keys
68
+ unless unknown.empty?
69
+ raise ArgumentError, "unknown probes: #{unknown.join(', ')}"
70
+ end
71
+
72
+ probes.each do |p|
73
+ require available[p.to_s]
74
+ end
75
+ end
76
+
77
+ def registered
78
+ @registered ||= {}
79
+ end
80
+
81
+ def require_hooks
82
+ @require_hooks ||= {}
83
+ end
84
+
85
+ def installed
86
+ @installed ||= {}
87
+ end
88
+
89
+ def register(name, *args)
90
+ if registered.key?(name)
91
+ raise "already registered: #{name}"
92
+ end
93
+
94
+ registered[name] = ProbeRegistration.new(name, *args)
95
+
96
+ true
97
+ end
98
+
99
+ def require_hook(require_path)
100
+ each_by_require_path(require_path) do |registration|
101
+ # Double check constant is available
102
+ if registration.constant_available?
103
+ install_probe(registration)
104
+
105
+ # Don't need this to be called again
106
+ unregister_require_hook(registration)
107
+ end
108
+ end
109
+ end
110
+
111
+ def register_require_hook(registration)
112
+ registration.require_paths.each do |p|
113
+ require_hooks[p] ||= []
114
+ require_hooks[p] << registration
115
+ end
116
+ end
117
+
118
+ def unregister_require_hook(registration)
119
+ registration.require_paths.each do |p|
120
+ require_hooks[p].delete(registration)
121
+ require_hooks.delete(p) if require_hooks[p].empty?
122
+ end
123
+ end
124
+
125
+ def each_by_require_path(require_path)
126
+ return unless require_hooks.key?(require_path)
127
+
128
+ # dup because we may be mutating the array
129
+ require_hooks[require_path].dup.each do |registration|
130
+ yield registration
131
+ end
132
+ end
133
+ end
134
+
135
+ add_path(File.expand_path("./probes", __dir__))
136
+ end
137
+ end
138
+
139
+ # @api private
140
+ module Kernel
141
+ # Unfortunately, we can't use prepend here, in part because RubyGems changes require with an alias
142
+ alias require_without_sk require
143
+
144
+ def require(name)
145
+ require_without_sk(name).tap do
146
+ begin
147
+ Skylight::Probes.require_hook(name)
148
+ rescue Exception => e # rubocop:disable Lint/SuppressedException
149
+ warn("[SKYLIGHT] Rescued exception in require hook", e)
150
+ end
151
+ end
152
+ end
153
+ end
@@ -0,0 +1,48 @@
1
+ module Skylight
2
+ module Probes
3
+ module ActionController
4
+ class Probe
5
+ def install
6
+ # Prepending doesn't work here since this a module that's already been included
7
+ ::ActionController::Instrumentation.class_eval do
8
+ private
9
+
10
+ alias_method :append_info_to_payload_without_sk, :append_info_to_payload
11
+ def append_info_to_payload(payload)
12
+ append_info_to_payload_without_sk(payload)
13
+
14
+ payload[:sk_rendered_format] = sk_rendered_mime.try(:ref)
15
+ payload[:sk_variant] = request.respond_to?(:variant) ? request.variant : nil
16
+ end
17
+
18
+ def sk_rendered_mime
19
+ if respond_to?(:media_type)
20
+ mt = media_type
21
+ return mt && Mime::Type.lookup(mt)
22
+ end
23
+
24
+ if content_type.is_a?(Mime::Type)
25
+ content_type
26
+ elsif content_type.respond_to?(:to_s)
27
+ type_str = content_type.to_s.split(";").first
28
+ Mime::Type.lookup(type_str) unless type_str.blank?
29
+ elsif respond_to?(:rendered_format) && rendered_format
30
+ rendered_format
31
+ end
32
+ rescue
33
+ # There are cases in which actionpack can return
34
+ # a stringified representation of a Mime::NullType instance,
35
+ # which is invalid for a number of reasons. This string raises
36
+ # errors when piped through Mime::Type.lookup, so it's probably
37
+ # best to just return nil in those cases.
38
+ nil
39
+ end
40
+ end
41
+ end
42
+ end
43
+ end
44
+
45
+ register(:action_controller, "ActionController::Instrumentation", "action_controller/metal/instrumentation",
46
+ ActionController::Probe.new)
47
+ end
48
+ end
@@ -0,0 +1,2 @@
1
+ Skylight.probe("action_dispatch/request_id")
2
+ Skylight.probe("action_dispatch/routing/route_set")
@@ -0,0 +1,29 @@
1
+ module Skylight
2
+ module Probes
3
+ module ActionDispatch
4
+ module RequestId
5
+ module Instrumentation
6
+ def call(env)
7
+ @skylight_request_id = env["skylight.request_id"]
8
+ super
9
+ end
10
+
11
+ private
12
+
13
+ def internal_request_id
14
+ @skylight_request_id || super
15
+ end
16
+ end
17
+
18
+ class Probe
19
+ def install
20
+ ::ActionDispatch::RequestId.prepend(Instrumentation)
21
+ end
22
+ end
23
+ end
24
+ end
25
+
26
+ register(:action_dispatch, "ActionDispatch::RequestId", "action_dispatch/middleware/request_id",
27
+ ActionDispatch::RequestId::Probe.new)
28
+ end
29
+ end
@@ -0,0 +1,28 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Skylight
4
+ module Probes
5
+ module ActionDispatch
6
+ module Routing
7
+ module RouteSet
8
+ module Instrumentation
9
+ def call(env)
10
+ ActiveSupport::Notifications.instrument("route_set.action_dispatch") do
11
+ super
12
+ end
13
+ end
14
+ end
15
+
16
+ class Probe
17
+ def install
18
+ ::ActionDispatch::Routing::RouteSet.prepend(Instrumentation)
19
+ end
20
+ end
21
+ end
22
+ end
23
+ end
24
+
25
+ register(:rails_router, "ActionDispatch::Routing::RouteSet", "action_dispatch/routing/route_set",
26
+ ActionDispatch::Routing::RouteSet::Probe.new)
27
+ end
28
+ end
@@ -0,0 +1,43 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Skylight
4
+ module Probes
5
+ module ActionView
6
+ module Instrumentation
7
+ def render_with_layout(*args) #:nodoc:
8
+ path, locals = case args.length
9
+ when 2
10
+ args
11
+ when 4
12
+ # Rails > 6.0.0.beta3 arguments are (view, template, path, locals)
13
+ [args[2], args[3]]
14
+ end
15
+
16
+ layout = nil
17
+
18
+ if path
19
+ layout = find_layout(path, locals.keys, [formats.first])
20
+ end
21
+
22
+ if layout
23
+ ActiveSupport::Notifications.instrument("render_template.action_view", identifier: layout.identifier) do
24
+ super
25
+ end
26
+ else
27
+ super
28
+ end
29
+ end
30
+ end
31
+
32
+ class Probe
33
+ def install
34
+ return if ::ActionView.gem_version >= Gem::Version.new("6.1.0.alpha")
35
+
36
+ ::ActionView::TemplateRenderer.prepend(Instrumentation)
37
+ end
38
+ end
39
+ end
40
+
41
+ register(:action_view, "ActionView::TemplateRenderer", "action_view", ActionView::Probe.new)
42
+ end
43
+ end