skylight 0.3.21 → 0.4.0.alpha1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +0 -4
  3. data/ext/extconf.rb +92 -47
  4. data/ext/libskylight.yml +4 -4
  5. data/ext/skylight_native.c +248 -286
  6. data/lib/skylight.rb +19 -114
  7. data/lib/skylight/api.rb +1 -1
  8. data/lib/skylight/config.rb +176 -146
  9. data/lib/skylight/data/cacert.pem +717 -719
  10. data/lib/skylight/formatters/http.rb +1 -1
  11. data/lib/skylight/instrumenter.rb +28 -35
  12. data/lib/skylight/native.rb +58 -72
  13. data/lib/skylight/normalizers.rb +0 -1
  14. data/lib/skylight/normalizers/active_record/sql.rb +0 -4
  15. data/lib/skylight/probes/excon/middleware.rb +3 -1
  16. data/lib/skylight/probes/net_http.rb +3 -1
  17. data/lib/skylight/subscriber.rb +0 -4
  18. data/lib/skylight/trace.rb +189 -0
  19. data/lib/skylight/util.rb +10 -12
  20. data/lib/skylight/util/hostname.rb +17 -0
  21. data/lib/skylight/util/http.rb +33 -36
  22. data/lib/skylight/util/logging.rb +20 -1
  23. data/lib/skylight/util/multi_io.rb +21 -0
  24. data/lib/skylight/util/native_ext_fetcher.rb +83 -69
  25. data/lib/skylight/util/platform.rb +67 -0
  26. data/lib/skylight/util/ssl.rb +50 -0
  27. data/lib/skylight/version.rb +1 -1
  28. metadata +9 -34
  29. data/ext/rust_support/ruby.h +0 -93
  30. data/ext/skylight.h +0 -85
  31. data/ext/skylight.map +0 -4
  32. data/ext/test/extconf.rb +0 -18
  33. data/ext/test/skylight_native_test.c +0 -82
  34. data/ext/test/skylight_test.h +0 -20
  35. data/lib/skylight/formatters.rb +0 -6
  36. data/lib/skylight/messages.rb +0 -21
  37. data/lib/skylight/messages/error.rb +0 -15
  38. data/lib/skylight/messages/hello.rb +0 -13
  39. data/lib/skylight/messages/trace.rb +0 -179
  40. data/lib/skylight/messages/trace_envelope.rb +0 -19
  41. data/lib/skylight/metrics.rb +0 -9
  42. data/lib/skylight/metrics/ewma.rb +0 -69
  43. data/lib/skylight/metrics/meter.rb +0 -58
  44. data/lib/skylight/metrics/process_cpu_gauge.rb +0 -65
  45. data/lib/skylight/metrics/process_mem_gauge.rb +0 -34
  46. data/lib/skylight/util/conversions.rb +0 -9
  47. data/lib/skylight/util/queue.rb +0 -96
  48. data/lib/skylight/util/task.rb +0 -172
  49. data/lib/skylight/util/uniform_sample.rb +0 -63
  50. data/lib/skylight/worker.rb +0 -19
  51. data/lib/skylight/worker/builder.rb +0 -73
  52. data/lib/skylight/worker/collector.rb +0 -274
  53. data/lib/skylight/worker/connection.rb +0 -87
  54. data/lib/skylight/worker/connection_set.rb +0 -56
  55. data/lib/skylight/worker/embedded.rb +0 -24
  56. data/lib/skylight/worker/metrics_reporter.rb +0 -104
  57. data/lib/skylight/worker/server.rb +0 -336
  58. data/lib/skylight/worker/standalone.rb +0 -421
@@ -14,4 +14,4 @@ module Skylight
14
14
  end
15
15
  end
16
16
  end
17
- end
17
+ end
@@ -59,16 +59,27 @@ module Skylight
59
59
  end
60
60
  end
61
61
 
62
+ at_exit { stop! }
63
+
62
64
  attr_reader :config, :gc, :trace_info
63
65
 
64
- def initialize(config)
65
- if Hash === config
66
+ def self.new(config)
67
+ if config.nil?
68
+ raise ArgumentError, "config is required"
69
+ elsif Hash === config
66
70
  config = Config.new(config)
67
71
  end
68
72
 
73
+ config.validate!
74
+
75
+ inst = native_new(config.to_env)
76
+ inst.send(:initialize, config)
77
+ inst
78
+ end
79
+
80
+ def initialize(config)
69
81
  @gc = config.gc
70
82
  @config = config
71
- @worker = config.worker.build
72
83
  @subscriber = Subscriber.new(config, self)
73
84
 
74
85
  @trace_info = @config[:trace_info] || TraceInfo.new
@@ -84,8 +95,6 @@ module Skylight
84
95
  end
85
96
 
86
97
  def start!
87
- return unless config
88
-
89
98
  # Warn if there was an error installing Skylight.
90
99
  # We do this here since we can't report these issues via Gem install without stopping install entirely.
91
100
  Skylight.check_install_errors(config)
@@ -98,48 +107,37 @@ module Skylight
98
107
  t { "starting instrumenter" }
99
108
  @config.validate!
100
109
 
101
- case @config.validate_token
102
- when :ok
103
- # Good to go
104
- when :unknown
105
- log_warn "unable to validate authentication token"
106
- else
107
- raise ConfigError, "authentication token is invalid"
110
+ t { "starting native instrumenter" }
111
+ unless native_start
112
+ warn "failed to start instrumenter"
113
+ return
108
114
  end
109
115
 
110
116
  @config.gc.enable
111
-
112
- unless @worker.spawn
113
- log_error "failed to spawn worker"
114
- return nil
115
- end
116
-
117
117
  @subscriber.register!
118
118
 
119
119
  self
120
120
 
121
121
  rescue Exception => e
122
122
  log_error "failed to start instrumenter; msg=%s; config=%s", e.message, @config.inspect
123
+ t { e.backtrace.join("\n") }
123
124
  nil
124
125
  end
125
126
 
126
127
  def shutdown
127
128
  @subscriber.unregister!
128
- @worker.shutdown
129
+ native_stop
129
130
  end
130
131
 
131
132
  def trace(endpoint, cat, title=nil, desc=nil, annot=nil)
132
133
  # If a trace is already in progress, continue with that one
133
134
  if trace = @trace_info.current
134
- t { "already tracing" }
135
135
  return yield(trace) if block_given?
136
136
  return trace
137
137
  end
138
138
 
139
- t { "starting trace" }
140
-
141
139
  begin
142
- trace = Messages::Trace::Builder.new(self, endpoint, Util::Clock.nanos, cat, title, desc, annot)
140
+ trace = Trace.new(self, endpoint, Util::Clock.nanos, cat, title, desc, annot)
143
141
  rescue Exception => e
144
142
  log_error e.message
145
143
  t { e.backtrace.join("\n") }
@@ -154,6 +152,7 @@ module Skylight
154
152
 
155
153
  ensure
156
154
  @trace_info.current = nil
155
+ t { "submitting trace" }
157
156
  trace.submit
158
157
  end
159
158
  end
@@ -231,20 +230,14 @@ module Skylight
231
230
  end
232
231
  end
233
232
 
234
- def error(type, description, details=nil)
235
- t { fmt "processing error; type=%s; description=%s", type, description }
236
-
237
- message = Skylight::Messages::Error.build(type, description, details && details.to_json)
238
-
239
- unless @worker.submit(message)
240
- warn "failed to submit error to worker"
241
- end
242
- end
243
-
244
233
  def process(trace)
245
234
  t { fmt "processing trace" }
246
- unless @worker.submit(trace)
247
- warn "failed to submit trace to worker"
235
+ begin
236
+ native_submit_trace(trace)
237
+ true
238
+ rescue => e
239
+ warn "failed to submit trace to worker; err=%s", e
240
+ false
248
241
  end
249
242
  end
250
243
 
@@ -1,90 +1,76 @@
1
- module Skylight
2
-
3
- if native?
4
-
5
- # @api private
6
- class Hello
7
- DIGITS = /^\s*\d+\s*$/
8
-
9
- alias serialize native_serialize
10
- alias version native_get_version
11
-
12
- class << self
13
- alias deserialize native_load
14
- end
1
+ require 'skylight/util/platform'
15
2
 
16
- def cmd
17
- native_cmd_length.times.map do |offset|
18
- native_cmd_get(offset)
19
- end
20
- end
3
+ module Skylight
4
+ # @api private
5
+ # Whether or not the native extension is present
6
+ @@has_native_ext = false
21
7
 
22
- def newer?(other = VERSION)
23
- other = split(other)
24
- curr = split(version)
8
+ def self.native?
9
+ @@has_native_ext
10
+ end
25
11
 
26
- [other.length, curr.length].max.times do |i|
27
- next if other[i] == curr[i]
28
- return true unless other[i]
12
+ def self.libskylight_path
13
+ ENV['SKYLIGHT_LIB_PATH'] || File.expand_path("../native/#{Util::Platform.tuple}", __FILE__)
14
+ end
29
15
 
30
- if other[i] =~ DIGITS
31
- if curr[i] =~ DIGITS
32
- other_i = other[i].to_i
33
- curr_i = curr[i].to_i
16
+ begin
17
+ unless ENV["SKYLIGHT_DISABLE_AGENT"]
18
+ lib = "#{libskylight_path}/libskylight.#{Util::Platform.libext}"
34
19
 
35
- next if other_i == curr_i
20
+ if File.exist?(lib)
21
+ # First attempt to require the native extension
22
+ require 'skylight_native'
36
23
 
37
- return curr_i > other_i
38
- else
39
- return false
40
- end
41
- else
42
- if curr[i] =~ DIGITS
43
- return true
44
- else
45
- next if curr[i] == other[i]
46
- return curr[i] > other[i]
47
- end
48
- end
49
- end
24
+ # Attempt to link the dylib
25
+ load_libskylight(lib)
50
26
 
51
- false
27
+ # If nothing was thrown, then the native extension is present
28
+ @@has_native_ext = true
52
29
  end
53
-
54
- private
55
-
56
- def split(v)
57
- v.split('.')
58
- end
59
-
60
30
  end
31
+ rescue LoadError => e
32
+ raise if ENV.key?("SKYLIGHT_REQUIRED")
33
+ end
61
34
 
62
- # @api private
63
- class Error
64
- alias serialize native_serialize
65
- alias type native_get_group
66
- alias description native_get_description
67
- alias details native_get_details
68
-
69
- class << self
70
- alias deserialize native_load
35
+ unless Skylight.native?
36
+ class Instrumenter
37
+ def self.native_new(*args)
38
+ allocate
71
39
  end
72
40
  end
41
+ end
73
42
 
74
- # @api private
75
- class Trace
76
- alias serialize native_serialize
77
- end
78
-
79
- # @api private
80
- class Batch
81
- alias serialize native_serialize
43
+ # @api private
44
+ def self.check_install_errors(config)
45
+ # Note: An unsupported arch doesn't count as an error.
46
+ install_log = File.expand_path("../../ext/install.log", __FILE__)
47
+
48
+ if File.exist?(install_log) && File.read(install_log) =~ /ERROR/
49
+ config.alert_logger.error \
50
+ "[SKYLIGHT] [#{Skylight::VERSION}] The Skylight native extension failed to install. " \
51
+ "Please check #{install_log} and notify support@skylight.io." \
52
+ "The missing extension will not affect the functioning of your application."
82
53
  end
83
54
  end
84
55
 
85
56
  # @api private
86
- # We have to declare this again due to a YARD bug
87
- # https://github.com/lsegal/yard/issues/761
88
- class Hello; end
89
-
57
+ def self.warn_skylight_native_missing(config)
58
+ # TODO: Dumping the error messages this way is pretty hacky
59
+ is_rails = defined?(Rails)
60
+ env_name = is_rails ? Rails.env : "development"
61
+
62
+ if env_name == "development" || env_name == "test"
63
+ config.alert_logger.warn \
64
+ "[SKYLIGHT] [#{Skylight::VERSION}] Running Skylight in #{env_name} mode. " \
65
+ "No data will be reported until you deploy your app.\n" \
66
+ "(To disable this message, set `alert_log_file` in your config.)"
67
+ else
68
+ config.alert_logger.error \
69
+ "[SKYLIGHT] [#{Skylight::VERSION}] The Skylight native extension for your platform wasn't found. " \
70
+ "The monitoring portion of Skylight is only supported on production servers running 32- or " \
71
+ "64-bit Linux. The missing extension will not affect the functioning of your application " \
72
+ "and you can continue local development without data being reported. If you are on a " \
73
+ "supported platform, please contact support at support@skylight.io."
74
+ end
75
+ end
90
76
  end
@@ -76,7 +76,6 @@ module Skylight
76
76
  path[start, path.size]
77
77
  end
78
78
  else
79
- annotations[:skylight_error] = ["absolute_path", path]
80
79
  "Absolute Path"
81
80
  end
82
81
  end
@@ -32,10 +32,6 @@ module Skylight
32
32
  sql: sql,
33
33
  binds: binds,
34
34
  }
35
- else
36
- annotations = {
37
- skylight_error: error
38
- }
39
35
  end
40
36
 
41
37
  [ name, title, sql, annotations ]
@@ -1,3 +1,5 @@
1
+ require 'skylight/formatters/http'
2
+
1
3
  module Skylight
2
4
  module Probes
3
5
  module Excon
@@ -60,4 +62,4 @@ module Skylight
60
62
  end
61
63
  end
62
64
  end
63
- end
65
+ end
@@ -1,3 +1,5 @@
1
+ require 'skylight/formatters/http'
2
+
1
3
  module Skylight
2
4
  module Probes
3
5
  module NetHTTP
@@ -39,4 +41,4 @@ module Skylight
39
41
 
40
42
  register("Net::HTTP", "net/http", NetHTTP::Probe.new)
41
43
  end
42
- end
44
+ end
@@ -42,10 +42,6 @@ module Skylight
42
42
 
43
43
  cat, title, desc, annot = normalize(trace, name, payload)
44
44
 
45
- if cat != :skip && error = annot.delete(:skylight_error)
46
- @instrumenter.error(*error)
47
- end
48
-
49
45
  unless cat == :skip
50
46
  span = trace.instrument(cat, title, desc, annot)
51
47
  end
@@ -0,0 +1,189 @@
1
+ module Skylight
2
+ class Trace
3
+ GC_CAT = 'noise.gc'.freeze
4
+
5
+ include Util::Logging
6
+
7
+ attr_reader :endpoint, :notifications
8
+
9
+ def self.new(instrumenter, endpoint, start, cat, title = nil, desc = nil, annot = nil)
10
+ inst = native_new(normalize_time(start), "TODO", endpoint)
11
+ inst.send(:initialize, instrumenter, cat, title, desc, annot)
12
+ inst.endpoint = endpoint
13
+ inst
14
+ end
15
+
16
+ # TODO: Move this into native
17
+ def self.normalize_time(time)
18
+ # At least one customer has extensions that cause integer division to produce rationals.
19
+ # Since the native code expects an integer, we force it again.
20
+ (time.to_i / 100_000).to_i
21
+ end
22
+
23
+ def initialize(instrumenter, cat, title, desc, annot)
24
+ raise ArgumentError, 'instrumenter is required' unless instrumenter
25
+
26
+ @instrumenter = instrumenter
27
+ @submitted = false
28
+ @broken = false
29
+
30
+ @notifications = []
31
+
32
+ if Hash === title
33
+ annot = title
34
+ title = desc = nil
35
+ elsif Hash === desc
36
+ annot = desc
37
+ desc = nil
38
+ end
39
+
40
+ # create the root node
41
+ @root = native_start_span(native_get_started_at, cat)
42
+ native_span_set_title(@root, title) if title
43
+ native_span_set_description(@root, desc) if desc
44
+
45
+ @gc = config.gc.track unless ENV.key?("SKYLIGHT_DISABLE_GC_TRACKING")
46
+ end
47
+
48
+ def endpoint=(value)
49
+ @endpoint = value
50
+ native_set_endpoint(value)
51
+ value
52
+ end
53
+
54
+ def config
55
+ @instrumenter.config
56
+ end
57
+
58
+ def record(cat, title=nil, desc=nil, annot=nil)
59
+ @return if @broken
60
+
61
+ if Hash === title
62
+ annot = title
63
+ title = desc = nil
64
+ elsif Hash === desc
65
+ annot = desc
66
+ desc = nil
67
+ end
68
+
69
+ title.freeze if title.is_a?(String)
70
+ desc.freeze if desc.is_a?(String)
71
+
72
+ desc = @instrumenter.limited_description(desc)
73
+
74
+ time = Util::Clock.nanos - gc_time
75
+
76
+ stop(start(time, cat, title, desc), time)
77
+
78
+ nil
79
+ rescue => e
80
+ error "failed to record span; msg=%s", e.message
81
+ @broken = true
82
+ nil
83
+ end
84
+
85
+ def instrument(cat, title=nil, desc=nil, annot=nil)
86
+ return if @broken
87
+ t { "instrument: #{cat}, #{title}" }
88
+
89
+ if Hash === title
90
+ annot = title
91
+ title = desc = nil
92
+ elsif Hash === desc
93
+ annot = desc
94
+ desc = nil
95
+ end
96
+
97
+ title.freeze if title.is_a?(String)
98
+ desc.freeze if desc.is_a?(String)
99
+
100
+ original_desc = desc
101
+ now = Util::Clock.nanos
102
+ desc = @instrumenter.limited_description(desc)
103
+
104
+ if desc == Instrumenter::TOO_MANY_UNIQUES
105
+ debug "[SKYLIGHT] [#{Skylight::VERSION}] A payload description produced <too many uniques>"
106
+ debug "original desc=%s", original_desc
107
+ debug "cat=%s, title=%s, desc=%s, annot=%s", cat, title, desc, annot.inspect
108
+ end
109
+
110
+ start(now - gc_time, cat, title, desc, annot)
111
+ rescue => e
112
+ error "failed to instrument span; msg=%s", e.message
113
+ @broken = true
114
+ nil
115
+ end
116
+
117
+ def done(span)
118
+ return unless span
119
+ return if @broken
120
+ stop(span, Util::Clock.nanos - gc_time)
121
+ rescue => e
122
+ error "failed to close span; msg=%s", e.message
123
+ @broken = true
124
+ nil
125
+ end
126
+
127
+ def release
128
+ return unless @instrumenter.current_trace == self
129
+ @instrumenter.current_trace = nil
130
+ end
131
+
132
+ def traced
133
+ time = gc_time
134
+ now = Util::Clock.nanos
135
+
136
+ if time > 0
137
+ t { fmt "tracking GC time; duration=%d", time }
138
+ stop(start(now - time, GC_CAT, nil, nil, {}), now)
139
+ end
140
+
141
+ stop(@root, now)
142
+ end
143
+
144
+ def submit
145
+ return if @broken
146
+
147
+ t { "submitting trace" }
148
+
149
+ if @submitted
150
+ t { "already submitted" }
151
+ return
152
+ end
153
+
154
+ release
155
+ @submitted = true
156
+
157
+ traced
158
+
159
+ @instrumenter.process(self)
160
+ rescue Exception => e
161
+ error e
162
+ t { e.backtrace.join("\n") }
163
+ end
164
+
165
+ private
166
+
167
+ def start(time, cat, title, desc, annot=nil)
168
+ span(self.class.normalize_time(time), cat, title, desc, annot)
169
+ end
170
+
171
+ def stop(span, time)
172
+ native_stop_span(span, self.class.normalize_time(time))
173
+ nil
174
+ end
175
+
176
+ def span(time, cat, title=nil, desc=nil, annot=nil)
177
+ sp = native_start_span(time, cat.to_s)
178
+ native_span_set_title(sp, title.to_s) if title
179
+ native_span_set_description(sp, desc.to_s) if desc
180
+ sp
181
+ end
182
+
183
+ def gc_time
184
+ return 0 unless @gc
185
+ @gc.update
186
+ @gc.time
187
+ end
188
+ end
189
+ end