skylight 4.2.1 → 5.0.0.beta

Sign up to get free protection for your applications and to get access to all the features.
Files changed (112) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +27 -0
  3. data/CONTRIBUTING.md +1 -7
  4. data/ext/extconf.rb +4 -3
  5. data/ext/libskylight.yml +5 -6
  6. data/ext/skylight_native.c +24 -100
  7. data/lib/skylight.rb +204 -14
  8. data/lib/skylight/api.rb +7 -3
  9. data/lib/skylight/cli.rb +4 -3
  10. data/lib/skylight/cli/doctor.rb +3 -2
  11. data/lib/skylight/cli/merger.rb +6 -4
  12. data/lib/skylight/config.rb +603 -126
  13. data/lib/skylight/deprecation.rb +15 -0
  14. data/lib/skylight/errors.rb +17 -2
  15. data/lib/skylight/extensions.rb +99 -0
  16. data/lib/skylight/extensions/source_location.rb +249 -0
  17. data/lib/skylight/fanout.rb +0 -0
  18. data/lib/skylight/formatters/http.rb +19 -0
  19. data/lib/skylight/gc.rb +109 -0
  20. data/lib/skylight/helpers.rb +18 -2
  21. data/lib/skylight/instrumenter.rb +325 -15
  22. data/lib/skylight/middleware.rb +138 -1
  23. data/lib/skylight/native.rb +51 -1
  24. data/lib/skylight/native_ext_fetcher.rb +2 -1
  25. data/lib/skylight/normalizers.rb +151 -0
  26. data/lib/skylight/normalizers/action_controller/process_action.rb +69 -0
  27. data/lib/skylight/normalizers/action_controller/send_file.rb +50 -0
  28. data/lib/skylight/normalizers/action_dispatch/process_middleware.rb +22 -0
  29. data/lib/skylight/normalizers/action_dispatch/route_set.rb +27 -0
  30. data/lib/skylight/normalizers/action_view/render_collection.rb +24 -0
  31. data/lib/skylight/normalizers/action_view/render_layout.rb +25 -0
  32. data/lib/skylight/normalizers/action_view/render_partial.rb +23 -0
  33. data/lib/skylight/normalizers/action_view/render_template.rb +23 -0
  34. data/lib/skylight/normalizers/active_job/perform.rb +81 -0
  35. data/lib/skylight/normalizers/active_model_serializers/render.rb +28 -0
  36. data/lib/skylight/normalizers/active_record/instantiation.rb +16 -0
  37. data/lib/skylight/normalizers/active_record/sql.rb +12 -0
  38. data/lib/skylight/normalizers/active_storage.rb +30 -0
  39. data/lib/skylight/normalizers/active_support/cache.rb +22 -0
  40. data/lib/skylight/normalizers/active_support/cache_clear.rb +16 -0
  41. data/lib/skylight/normalizers/active_support/cache_decrement.rb +16 -0
  42. data/lib/skylight/normalizers/active_support/cache_delete.rb +16 -0
  43. data/lib/skylight/normalizers/active_support/cache_exist.rb +16 -0
  44. data/lib/skylight/normalizers/active_support/cache_fetch_hit.rb +16 -0
  45. data/lib/skylight/normalizers/active_support/cache_generate.rb +16 -0
  46. data/lib/skylight/normalizers/active_support/cache_increment.rb +16 -0
  47. data/lib/skylight/normalizers/active_support/cache_read.rb +16 -0
  48. data/lib/skylight/normalizers/active_support/cache_read_multi.rb +16 -0
  49. data/lib/skylight/normalizers/active_support/cache_write.rb +16 -0
  50. data/lib/skylight/normalizers/coach/handler_finish.rb +46 -0
  51. data/lib/skylight/normalizers/coach/middleware_finish.rb +33 -0
  52. data/lib/skylight/normalizers/couch_potato/query.rb +20 -0
  53. data/lib/skylight/normalizers/data_mapper/sql.rb +12 -0
  54. data/lib/skylight/normalizers/default.rb +32 -0
  55. data/lib/skylight/normalizers/elasticsearch/request.rb +20 -0
  56. data/lib/skylight/normalizers/faraday/request.rb +40 -0
  57. data/lib/skylight/normalizers/grape/endpoint.rb +34 -0
  58. data/lib/skylight/normalizers/grape/endpoint_render.rb +25 -0
  59. data/lib/skylight/normalizers/grape/endpoint_run.rb +41 -0
  60. data/lib/skylight/normalizers/grape/endpoint_run_filters.rb +22 -0
  61. data/lib/skylight/normalizers/grape/format_response.rb +20 -0
  62. data/lib/skylight/normalizers/graphiti/render.rb +22 -0
  63. data/lib/skylight/normalizers/graphiti/resolve.rb +31 -0
  64. data/lib/skylight/normalizers/graphql/base.rb +131 -0
  65. data/lib/skylight/normalizers/render.rb +81 -0
  66. data/lib/skylight/normalizers/sequel/sql.rb +12 -0
  67. data/lib/skylight/normalizers/sql.rb +44 -0
  68. data/lib/skylight/probes.rb +153 -0
  69. data/lib/skylight/probes/action_controller.rb +48 -0
  70. data/lib/skylight/probes/action_dispatch.rb +2 -0
  71. data/lib/skylight/probes/action_dispatch/request_id.rb +29 -0
  72. data/lib/skylight/probes/action_dispatch/routing/route_set.rb +28 -0
  73. data/lib/skylight/probes/action_view.rb +43 -0
  74. data/lib/skylight/probes/active_job.rb +29 -0
  75. data/lib/skylight/probes/active_job_enqueue.rb +37 -0
  76. data/lib/skylight/probes/active_model_serializers.rb +54 -0
  77. data/lib/skylight/probes/delayed_job.rb +62 -0
  78. data/lib/skylight/probes/elasticsearch.rb +38 -0
  79. data/lib/skylight/probes/excon.rb +25 -0
  80. data/lib/skylight/probes/excon/middleware.rb +66 -0
  81. data/lib/skylight/probes/faraday.rb +23 -0
  82. data/lib/skylight/probes/graphql.rb +43 -0
  83. data/lib/skylight/probes/httpclient.rb +44 -0
  84. data/lib/skylight/probes/middleware.rb +125 -0
  85. data/lib/skylight/probes/mongo.rb +163 -0
  86. data/lib/skylight/probes/mongoid.rb +13 -0
  87. data/lib/skylight/probes/net_http.rb +55 -0
  88. data/lib/skylight/probes/redis.rb +60 -0
  89. data/lib/skylight/probes/sequel.rb +33 -0
  90. data/lib/skylight/probes/sinatra.rb +63 -0
  91. data/lib/skylight/probes/sinatra_add_middleware.rb +10 -10
  92. data/lib/skylight/probes/tilt.rb +27 -0
  93. data/lib/skylight/railtie.rb +162 -18
  94. data/lib/skylight/sidekiq.rb +43 -0
  95. data/lib/skylight/subscriber.rb +110 -0
  96. data/lib/skylight/test.rb +146 -0
  97. data/lib/skylight/trace.rb +301 -10
  98. data/lib/skylight/user_config.rb +61 -0
  99. data/lib/skylight/util.rb +12 -0
  100. data/lib/skylight/util/allocation_free.rb +26 -0
  101. data/lib/skylight/util/clock.rb +56 -0
  102. data/lib/skylight/util/component.rb +5 -2
  103. data/lib/skylight/util/deploy.rb +4 -4
  104. data/lib/skylight/util/gzip.rb +20 -0
  105. data/lib/skylight/util/http.rb +4 -10
  106. data/lib/skylight/util/instrumenter_method.rb +26 -0
  107. data/lib/skylight/util/logging.rb +138 -0
  108. data/lib/skylight/util/lru_cache.rb +42 -0
  109. data/lib/skylight/vendor/cli/thor/rake_compat.rb +1 -1
  110. data/lib/skylight/version.rb +5 -1
  111. data/lib/skylight/vm/gc.rb +68 -0
  112. metadata +116 -17
@@ -0,0 +1,163 @@
1
+ module Skylight
2
+ module Probes
3
+ module Mongo
4
+ CAT = "db.mongo.command".freeze
5
+
6
+ class Probe
7
+ def install
8
+ ::Mongo::Monitoring::Global.subscribe(::Mongo::Monitoring::COMMAND, Subscriber.new)
9
+ end
10
+ end
11
+
12
+ class Subscriber
13
+ include Skylight::Util::Logging
14
+
15
+ COMMANDS = %i[insert find count distinct update findandmodify findAndModify delete aggregate].freeze
16
+
17
+ COMMAND_NAMES = {
18
+ findandmodify: "findAndModify".freeze
19
+ }.freeze
20
+
21
+ def initialize
22
+ @events = {}
23
+ end
24
+
25
+ def started(event)
26
+ begin_instrumentation(event)
27
+ end
28
+
29
+ def succeeded(event)
30
+ end_instrumentation(event)
31
+ end
32
+
33
+ def failed(event)
34
+ end_instrumentation(event)
35
+ end
36
+
37
+ # For logging
38
+ def config
39
+ Skylight.config
40
+ end
41
+
42
+ private
43
+
44
+ def begin_instrumentation(event)
45
+ return unless COMMANDS.include?(event.command_name.to_sym)
46
+
47
+ command_name = COMMAND_NAMES[event.command_name] || event.command_name.to_s
48
+
49
+ title = "#{event.database_name}.#{command_name}"
50
+
51
+ command = event.command
52
+
53
+ # Not sure if this will always exist
54
+ # Delete so the description will be less redundant
55
+ if (target = command[event.command_name])
56
+ title << " #{target}"
57
+ end
58
+
59
+ payload = {}
60
+
61
+ # Ruby Hashes are ordered based on insertion so do the most important ones first
62
+
63
+ add_value("key".freeze, command, payload)
64
+ add_bound("query".freeze, command, payload)
65
+ add_bound("filter".freeze, command, payload)
66
+ add_value("sort".freeze, command, payload)
67
+
68
+ if command_name == "findAndModify".freeze
69
+ add_bound("update".freeze, command, payload)
70
+ end
71
+
72
+ add_value("remove".freeze, command, payload)
73
+ add_value("new".freeze, command, payload)
74
+
75
+ if (updates = command["updates".freeze])
76
+ # AFAICT the gem generally just sends one item in the updates array
77
+ update = updates[0]
78
+ update_payload = {}
79
+ add_bound("q".freeze, update, update_payload)
80
+ add_bound("u".freeze, update, update_payload)
81
+ add_value("multi".freeze, update, update_payload)
82
+ add_value("upsert".freeze, update, update_payload)
83
+
84
+ payload["updates".freeze] = [update_payload]
85
+
86
+ if updates.length > 1
87
+ payload["updates".freeze] << "..."
88
+ end
89
+ end
90
+
91
+ if (deletes = command["deletes".freeze])
92
+ # AFAICT the gem generally just sends one item in the updates array
93
+ delete = deletes[0]
94
+ delete_payload = {}
95
+ add_bound("q".freeze, delete, delete_payload)
96
+ add_value("limit".freeze, delete, delete_payload)
97
+
98
+ payload["deletes".freeze] = [delete_payload]
99
+
100
+ if deletes.length > 1
101
+ payload["deletes".freeze] << "..."
102
+ end
103
+ end
104
+
105
+ if (pipeline = command["pipeline".freeze])
106
+ payload["pipeline".freeze] = pipeline.map { |segment| extract_binds(segment) }
107
+ end
108
+
109
+ # We're ignoring documents from insert because they could have completely inconsistent
110
+ # format which would make it hard to merge.
111
+
112
+ opts = {
113
+ category: CAT,
114
+ title: title,
115
+ description: payload.empty? ? nil : payload.to_json,
116
+ meta: { database: event.database_name }
117
+ }
118
+
119
+ @events[event.operation_id] = Skylight.instrument(opts)
120
+ rescue Exception => e
121
+ error "failed to begin instrumentation for Mongo; msg=%s", e.message
122
+ end
123
+
124
+ def end_instrumentation(event)
125
+ if (original_event = @events.delete(event.operation_id))
126
+ meta = {}
127
+ if event.is_a?(::Mongo::Monitoring::Event::CommandFailed)
128
+ meta[:exception] = ["CommandFailed", event.message]
129
+ end
130
+ Skylight.done(original_event, meta)
131
+ end
132
+ rescue Exception => e
133
+ error "failed to end instrumentation for Mongo; msg=%s", e.message
134
+ end
135
+
136
+ def add_value(key, command, payload)
137
+ if command.key?(key)
138
+ value = command[key]
139
+ payload[key] = value
140
+ end
141
+ end
142
+
143
+ def add_bound(key, command, payload)
144
+ if (value = command[key])
145
+ payload[key] = extract_binds(value)
146
+ end
147
+ end
148
+
149
+ def extract_binds(hash)
150
+ ret = {}
151
+
152
+ hash.each do |k, v|
153
+ ret[k] = v.is_a?(Hash) ? extract_binds(v) : "?".freeze
154
+ end
155
+
156
+ ret
157
+ end
158
+ end
159
+ end
160
+
161
+ register(:mongo, "Mongo", "mongo", Mongo::Probe.new)
162
+ end
163
+ end
@@ -0,0 +1,13 @@
1
+ module Skylight
2
+ module Probes
3
+ module Mongoid
4
+ class Probe
5
+ def install
6
+ Skylight::Probes.probe(:mongo)
7
+ end
8
+ end
9
+ end
10
+
11
+ register(:mongoid, "Mongoid", "mongoid", Mongoid::Probe.new)
12
+ end
13
+ end
@@ -0,0 +1,55 @@
1
+ require "skylight/formatters/http"
2
+
3
+ module Skylight
4
+ module Probes
5
+ module NetHTTP
6
+ module Instrumentation
7
+ def request(req, *)
8
+ return super if !started? || Probes::NetHTTP::Probe.disabled?
9
+
10
+ method = req.method
11
+
12
+ # req['host'] also includes special handling for default ports
13
+ host, port = req["host"] ? req["host"].split(":") : nil
14
+
15
+ # If we're connected with a persistent socket
16
+ host ||= address
17
+ port ||= port
18
+
19
+ path = req.path
20
+ scheme = use_ssl? ? "https" : "http"
21
+
22
+ # Contained in the path
23
+ query = nil
24
+
25
+ opts = Formatters::HTTP.build_opts(method, scheme, host, port, path, query)
26
+
27
+ Skylight.instrument(opts) { super }
28
+ end
29
+ end
30
+
31
+ # Probe for instrumenting Net::HTTP requests. Works by monkeypatching the default Net::HTTP#request method.
32
+ class Probe
33
+ DISABLED_KEY = :__skylight_net_http_disabled
34
+
35
+ def self.disable
36
+ state_was = Thread.current[DISABLED_KEY]
37
+ Thread.current[DISABLED_KEY] = true
38
+ yield
39
+ ensure
40
+ Thread.current[DISABLED_KEY] = state_was
41
+ end
42
+
43
+ def self.disabled?
44
+ !!Thread.current[DISABLED_KEY]
45
+ end
46
+
47
+ def install
48
+ Net::HTTP.prepend(Instrumentation)
49
+ end
50
+ end
51
+ end
52
+
53
+ register(:net_http, "Net::HTTP", "net/http", NetHTTP::Probe.new)
54
+ end
55
+ end
@@ -0,0 +1,60 @@
1
+ module Skylight
2
+ module Probes
3
+ module Redis
4
+ # Unfortunately, because of the nature of pipelining, there's no way for us to
5
+ # give a time breakdown on the individual items.
6
+
7
+ PIPELINED_OPTS = {
8
+ category: "db.redis.pipelined".freeze,
9
+ title: "PIPELINE".freeze
10
+ }.freeze
11
+
12
+ MULTI_OPTS = {
13
+ category: "db.redis.multi".freeze,
14
+ title: "MULTI".freeze
15
+ }.freeze
16
+
17
+ module ClientInstrumentation
18
+ def call(command, *)
19
+ command_name = command[0]
20
+
21
+ return super if command_name == :auth
22
+
23
+ opts = {
24
+ category: "db.redis.command",
25
+ title: command_name.upcase.to_s
26
+ }
27
+
28
+ Skylight.instrument(opts) { super }
29
+ end
30
+ end
31
+
32
+ module Instrumentation
33
+ def pipelined(*)
34
+ Skylight.instrument(PIPELINED_OPTS) { super }
35
+ end
36
+
37
+ def multi(*)
38
+ Skylight.instrument(MULTI_OPTS) { super }
39
+ end
40
+ end
41
+
42
+ class Probe
43
+ def install
44
+ version = defined?(::Redis::VERSION) ? Gem::Version.new(::Redis::VERSION) : nil
45
+
46
+ if !version || version < Gem::Version.new("3.0.0")
47
+ Skylight.error "The installed version of Redis doesn't support Middlewares. " \
48
+ "At least version 3.0.0 is required."
49
+ return
50
+ end
51
+
52
+ ::Redis::Client.prepend(ClientInstrumentation)
53
+ ::Redis.prepend(Instrumentation)
54
+ end
55
+ end
56
+ end
57
+
58
+ register(:redis, "Redis", "redis", Redis::Probe.new)
59
+ end
60
+ end
@@ -0,0 +1,33 @@
1
+ # Supports 3.12.0+
2
+ module Skylight
3
+ module Probes
4
+ module Sequel
5
+ class Probe
6
+ def install
7
+ require "sequel/database/logging"
8
+
9
+ method_name = ::Sequel::Database.method_defined?(:log_connection_yield) ? "log_connection_yield" : "log_yield"
10
+
11
+ mod = Module.new do
12
+ define_method method_name do |sql, *args, &block|
13
+ super(sql, *args) do
14
+ ::ActiveSupport::Notifications.instrument(
15
+ "sql.sequel",
16
+ sql: sql,
17
+ name: "SQL",
18
+ binds: args
19
+ ) do
20
+ block.call
21
+ end
22
+ end
23
+ end
24
+ end
25
+
26
+ ::Sequel::Database.prepend(mod)
27
+ end
28
+ end
29
+ end
30
+
31
+ register(:sequel, "Sequel", "sequel", Sequel::Probe.new)
32
+ end
33
+ end
@@ -0,0 +1,63 @@
1
+ module Skylight
2
+ module Probes
3
+ module Sinatra
4
+ module ClassInstrumentation
5
+ def compile!(verb, path, *)
6
+ super.tap do |_, _, keys_or_wrapper, wrapper|
7
+ wrapper ||= keys_or_wrapper
8
+
9
+ # Deal with the situation where the path is a regex, and the default behavior
10
+ # of Ruby stringification produces an unreadable mess
11
+ if path.is_a?(Regexp)
12
+ human_readable = "<sk-regex>%r{#{path.source}}</sk-regex>"
13
+ wrapper.instance_variable_set(:@route_name, "#{verb} #{human_readable}")
14
+ else
15
+ wrapper.instance_variable_set(:@route_name, "#{verb} #{path}")
16
+ end
17
+ end
18
+ end
19
+ end
20
+
21
+ module Instrumentation
22
+ def dispatch!(*)
23
+ super.tap do
24
+ if (trace = Skylight.instrumenter&.current_trace) && (route = env["sinatra.route"])
25
+ # Include the app's mount point (if available)
26
+ script_name = trace.instrumenter.config.sinatra_route_prefixes? && env["SCRIPT_NAME"]
27
+
28
+ trace.endpoint =
29
+ if script_name && !script_name.empty?
30
+ verb, path = route.split(" ", 2)
31
+ "#{verb} [#{script_name}]#{path}"
32
+ else
33
+ route
34
+ end
35
+ end
36
+ end
37
+ end
38
+
39
+ def compile_template(engine, data, options, *)
40
+ # Pass along a useful "virtual path" to Tilt. The Tilt probe will handle
41
+ # instrumenting correctly.
42
+ options[:sky_virtual_path] = data.is_a?(Symbol) ? data.to_s : "Inline template (#{engine})"
43
+
44
+ super
45
+ end
46
+ end
47
+
48
+ class Probe
49
+ def install
50
+ if ::Sinatra::VERSION < "1.4.0"
51
+ Skylight.error "Sinatra must be version 1.4.0 or greater."
52
+ return
53
+ end
54
+
55
+ ::Sinatra::Base.singleton_class.prepend(ClassInstrumentation)
56
+ ::Sinatra::Base.prepend(Instrumentation)
57
+ end
58
+ end
59
+ end
60
+
61
+ register(:sinatra, "Sinatra::Base", "sinatra/base", Sinatra::Probe.new)
62
+ end
63
+ end
@@ -1,20 +1,20 @@
1
1
  module Skylight
2
2
  module Probes
3
3
  module Sinatra
4
- class Probe
5
- def install
6
- class << ::Sinatra::Base
7
- alias_method :build_without_sk, :build
4
+ module Instrumentation
5
+ def build(*)
6
+ use Skylight::Middleware
7
+ super
8
+ end
9
+ end
8
10
 
9
- def build(*args, &block)
10
- use Skylight::Middleware
11
- build_without_sk(*args, &block)
12
- end
13
- end
11
+ class AddMiddlewareProbe
12
+ def install
13
+ ::Sinatra::Base.singleton_class.prepend(Instrumentation)
14
14
  end
15
15
  end
16
16
  end
17
17
 
18
- Skylight::Core::Probes.register(:sinatra_add_middleware, "Sinatra::Base", "sinatra/base", Sinatra::Probe.new)
18
+ register(:sinatra_add_middleware, "Sinatra::Base", "sinatra/base", Sinatra::AddMiddlewareProbe.new)
19
19
  end
20
20
  end
@@ -0,0 +1,27 @@
1
+ # Should support 0.2+, though not tested against older versions
2
+ module Skylight
3
+ module Probes
4
+ module Tilt
5
+ module Instrumentation
6
+ def render(*args, &block)
7
+ opts = {
8
+ category: "view.render.template",
9
+ title: options[:sky_virtual_path] || basename || "Unknown template name"
10
+ }
11
+
12
+ Skylight.instrument(opts) do
13
+ super(*args, &block)
14
+ end
15
+ end
16
+ end
17
+
18
+ class Probe
19
+ def install
20
+ ::Tilt::Template.prepend(Instrumentation)
21
+ end
22
+ end
23
+ end
24
+
25
+ register(:tilt, "Tilt::Template", "tilt/template", Tilt::Probe.new)
26
+ end
27
+ end