skylight 0.2.7 → 0.3.0.rc.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. checksums.yaml +4 -4
  2. data/ext/checksums.yml +3 -0
  3. data/ext/extconf.rb +90 -0
  4. data/ext/skylight.map +4 -0
  5. data/ext/skylight_native.c +613 -0
  6. data/lib/skylight.rb +50 -7
  7. data/lib/skylight/config.rb +7 -1
  8. data/lib/skylight/helpers.rb +2 -2
  9. data/lib/skylight/instrumenter.rb +9 -5
  10. data/lib/skylight/messages.rb +13 -13
  11. data/lib/skylight/messages/error.rb +10 -6
  12. data/lib/skylight/messages/hello.rb +4 -45
  13. data/lib/skylight/messages/trace.rb +62 -103
  14. data/lib/skylight/messages/trace_envelope.rb +19 -0
  15. data/lib/skylight/native.rb +80 -0
  16. data/lib/skylight/normalizers/process_action.rb +1 -3
  17. data/lib/skylight/probes.rb +1 -1
  18. data/lib/skylight/subscriber.rb +1 -1
  19. data/lib/skylight/util/clock.rb +13 -6
  20. data/lib/skylight/version.rb +1 -1
  21. data/lib/skylight/worker/builder.rb +1 -1
  22. data/lib/skylight/worker/collector.rb +20 -32
  23. data/lib/skylight/worker/connection.rb +2 -2
  24. data/lib/skylight/worker/embedded.rb +18 -0
  25. data/lib/skylight/worker/server.rb +3 -3
  26. data/lib/skylight/worker/standalone.rb +9 -10
  27. data/lib/sql_lexer.rb +0 -1
  28. data/lib/sql_lexer/lexer.rb +50 -4
  29. data/lib/sql_lexer/version.rb +1 -1
  30. metadata +19 -22
  31. data/lib/skylight/messages/annotation.rb +0 -13
  32. data/lib/skylight/messages/base.rb +0 -24
  33. data/lib/skylight/messages/batch.rb +0 -12
  34. data/lib/skylight/messages/endpoint.rb +0 -12
  35. data/lib/skylight/messages/event.rb +0 -12
  36. data/lib/skylight/messages/span.rb +0 -166
  37. data/lib/skylight/vendor/beefcake.rb +0 -292
  38. data/lib/skylight/vendor/beefcake/buffer.rb +0 -119
  39. data/lib/skylight/vendor/beefcake/decode.rb +0 -107
  40. data/lib/skylight/vendor/beefcake/encode.rb +0 -132
data/lib/skylight.rb CHANGED
@@ -2,6 +2,19 @@ require 'rbconfig'
2
2
  require 'socket'
3
3
  require 'skylight/version'
4
4
 
5
+ begin
6
+ unless ENV["SKYLIGHT_DISABLE_AGENT"]
7
+ require 'skylight_native'
8
+ require 'skylight/native'
9
+ has_native_ext = true
10
+ end
11
+ rescue LoadError
12
+ puts "[SKYLIGHT] The Skylight native extension wasn't found. Skylight is not running."
13
+ raise if ENV.key?("SKYLIGHT_REQUIRED")
14
+ end
15
+
16
+ if has_native_ext
17
+
5
18
  module Skylight
6
19
  TRACE_ENV_KEY = 'SKYLIGHT_ENABLE_TRACE_LOGS'.freeze
7
20
  STANDALONE_ENV_KEY = 'SKYLIGHT_STANDALONE'.freeze
@@ -32,7 +45,8 @@ module Skylight
32
45
  autoload :Worker, 'skylight/worker'
33
46
 
34
47
  module Util
35
- autoload :Clock, 'skylight/util/clock'
48
+ require 'skylight/util/clock'
49
+
36
50
  autoload :Gzip, 'skylight/util/gzip'
37
51
  autoload :HTTP, 'skylight/util/http'
38
52
  autoload :Inflector, 'skylight/util/inflector'
@@ -46,9 +60,6 @@ module Skylight
46
60
  autoload :HTTP, 'skylight/formatters/http'
47
61
  end
48
62
 
49
- # ==== Vendor ====
50
- autoload :Beefcake, 'skylight/vendor/beefcake'
51
-
52
63
  # ==== Exceptions ====
53
64
  class IpcProtoError < RuntimeError; end
54
65
  class WorkerStateError < RuntimeError; end
@@ -77,19 +88,24 @@ module Skylight
77
88
  Instrumenter.stop!(*args)
78
89
  end
79
90
 
80
- def self.trace(title=nil, desc=nil, annot=nil)
91
+ def self.trace(endpoint=nil, cat=nil, title=nil)
81
92
  unless inst = Instrumenter.instance
82
93
  return yield if block_given?
83
94
  return
84
95
  end
85
96
 
86
97
  if block_given?
87
- inst.trace(title, desc, annot) { yield }
98
+ inst.trace(endpoint, cat, title) { yield }
88
99
  else
89
- inst.trace(title, desc, annot)
100
+ inst.trace(endpoint, cat, title)
90
101
  end
91
102
  end
92
103
 
104
+ def self.done(span)
105
+ return unless inst = Instrumenter.instance
106
+ inst.done(span)
107
+ end
108
+
93
109
  def self.instrument(opts = DEFAULT_OPTIONS)
94
110
  unless inst = Instrumenter.instance
95
111
  return yield if block_given?
@@ -135,3 +151,30 @@ module Skylight
135
151
 
136
152
  require 'skylight/probes'
137
153
  end
154
+
155
+ else
156
+
157
+ module Skylight
158
+ def self.start!(*)
159
+ end
160
+
161
+ def self.stop!(*)
162
+ end
163
+
164
+ def self.trace(*)
165
+ yield if block_given?
166
+ end
167
+
168
+ def self.done(*)
169
+ end
170
+
171
+ def self.instrument(*)
172
+ yield if block_given?
173
+ end
174
+
175
+ def self.disable(*)
176
+ yield if block_given?
177
+ end
178
+ end
179
+
180
+ end
@@ -40,7 +40,9 @@ module Skylight
40
40
  'ACCOUNTS_SSL' => :'accounts.ssl',
41
41
  'ACCOUNTS_DEFLATE' => :'accounts.deflate',
42
42
  'ME_AUTHENTICATION' => :'me.authentication',
43
- 'ME_CREDENTIALS_PATH' => :'me.credentials_path' }
43
+ 'ME_CREDENTIALS_PATH' => :'me.credentials_path',
44
+ 'TEST_CONSTANT_FLUSH' => :'test.constant_flush',
45
+ 'TEST_IGNORE_TOKEN' => :'test.ignore_token' }
44
46
 
45
47
  # Default values for Skylight configuration keys
46
48
  DEFAULTS = {
@@ -261,6 +263,10 @@ authentication: #{self[:authentication]}
261
263
  get('test.constant_flush')
262
264
  end
263
265
 
266
+ def ignore_token?
267
+ get('test.ignore_token')
268
+ end
269
+
264
270
  def root
265
271
  self[:root] || Dir.pwd
266
272
  end
@@ -39,7 +39,7 @@ module Skylight
39
39
  title = (opts[:title] || title).to_s
40
40
  desc = opts[:description].to_s if opts[:description]
41
41
 
42
- klass.class_eval <<-RUBY
42
+ klass.class_eval <<-RUBY, __FILE__, __LINE__ + 1
43
43
  alias_method :"#{name}_before_instrument", :"#{name}"
44
44
 
45
45
  def #{name}(*args, &blk)
@@ -51,7 +51,7 @@ module Skylight
51
51
  begin
52
52
  #{name}_before_instrument(*args, &blk)
53
53
  ensure
54
- span.done if span
54
+ Skylight.done(span) if span
55
55
  end
56
56
  end
57
57
  RUBY
@@ -97,7 +97,7 @@ module Skylight
97
97
  end
98
98
 
99
99
  begin
100
- trace = Messages::Trace::Builder.new(self, endpoint, Util::Clock.micros, cat, title, desc, annot)
100
+ trace = Messages::Trace::Builder.new(self, endpoint, Util::Clock.nanos, cat, title, desc, annot)
101
101
  rescue Exception => e
102
102
  log_error e.message
103
103
  t { e.backtrace.join("\n") }
@@ -137,6 +137,11 @@ module Skylight
137
137
  self.class.match?(string, regex)
138
138
  end
139
139
 
140
+ def done(span)
141
+ return unless trace = @trace_info.current
142
+ trace.done(span)
143
+ end
144
+
140
145
  def instrument(cat, title=nil, desc=nil, annot=nil)
141
146
  unless trace = @trace_info.current
142
147
  return yield if block_given?
@@ -163,7 +168,7 @@ module Skylight
163
168
  begin
164
169
  yield sp
165
170
  ensure
166
- sp.done
171
+ trace.done(sp)
167
172
  end
168
173
  end
169
174
 
@@ -186,7 +191,7 @@ module Skylight
186
191
  def error(type, description, details=nil)
187
192
  t { fmt "processing error; type=%s; description=%s", type, description }
188
193
 
189
- message = Skylight::Messages::Error.new(type: type, description: description, details: details && details.to_json)
194
+ message = Skylight::Messages::Error.build(type, description, details && details.to_json)
190
195
 
191
196
  unless @worker.submit(message)
192
197
  warn "failed to submit error to worker"
@@ -194,8 +199,7 @@ module Skylight
194
199
  end
195
200
 
196
201
  def process(trace)
197
- t { fmt "processing trace; spans=%d; duration=%d",
198
- trace.spans.length, trace.spans[-1].duration }
202
+ t { fmt "processing trace" }
199
203
  unless @worker.submit(trace)
200
204
  warn "failed to submit trace to worker"
201
205
  end
@@ -1,20 +1,20 @@
1
1
  module Skylight
2
2
  module Messages
3
- def self.get(id)
4
- (@id_map ||= {})[id]
5
- end
6
-
7
- def self.set(id, klass)
8
- (@id_map ||= {})[id] = klass
9
- end
10
-
11
- require 'skylight/messages/annotation'
12
- require 'skylight/messages/event'
13
- require 'skylight/messages/span'
14
3
  require 'skylight/messages/trace'
15
- require 'skylight/messages/endpoint'
16
- require 'skylight/messages/batch'
17
4
  require 'skylight/messages/hello'
18
5
  require 'skylight/messages/error'
6
+ require 'skylight/messages/trace_envelope'
7
+
8
+ KLASS_TO_ID = {
9
+ Skylight::Hello => 1,
10
+ Skylight::Trace => 2,
11
+ Skylight::Error => 3
12
+ }
13
+
14
+ ID_TO_KLASS = {
15
+ 1 => Skylight::Hello,
16
+ 2 => Skylight::Messages::TraceEnvelope,
17
+ 3 => Skylight::Error
18
+ }
19
19
  end
20
20
  end
@@ -1,11 +1,15 @@
1
- require 'skylight/messages/base'
2
-
3
1
  module Skylight
4
2
  module Messages
5
- class Error < Base
6
- required :type, :string, 1
7
- required :description, :string, 2
8
- optional :details, :string, 3
3
+ class Error
4
+ def self.deserialize(buf)
5
+ decode(buf)
6
+ end
7
+
8
+ def self.build(group, description, details = nil)
9
+ Skylight::Error.native_new(group, description).tap do |error|
10
+ error.native_set_details(details) if details
11
+ end
12
+ end
9
13
  end
10
14
  end
11
15
  end
@@ -1,51 +1,10 @@
1
- require 'skylight/messages/base'
2
-
3
1
  module Skylight
4
2
  module Messages
5
- class Hello < Base
6
-
7
- DIGITS = /^\s*\d+\s*$/
8
-
9
- required :version, :string, 1
10
- optional :config, :uint32, 2
11
- repeated :cmd, :string, 3
12
-
13
- def newer?(other = VERSION)
14
- other = split(other)
15
- curr = split(version)
16
-
17
- [other.length, curr.length].max.times do |i|
18
- next if other[i] == curr[i]
19
- return true unless other[i]
20
-
21
- if other[i] =~ DIGITS
22
- if curr[i] =~ DIGITS
23
- other_i = other[i].to_i
24
- curr_i = curr[i].to_i
25
-
26
- next if other_i == curr_i
27
-
28
- return curr_i > other_i
29
- else
30
- return false
31
- end
32
- else
33
- if curr[i] =~ DIGITS
34
- return true
35
- else
36
- next if curr[i] == other[i]
37
- return curr[i] > other[i]
38
- end
39
- end
3
+ class Hello
4
+ def self.build(version, cmd=[])
5
+ Skylight::Hello.native_new(version, 0).tap do |hello|
6
+ cmd.each { |part| hello.native_add_cmd_part(part) }
40
7
  end
41
-
42
- false
43
- end
44
-
45
- private
46
-
47
- def split(v)
48
- v.split('.')
49
8
  end
50
9
 
51
10
  end
@@ -1,13 +1,6 @@
1
- require 'skylight/messages/base'
2
-
3
1
  module Skylight
4
2
  module Messages
5
- class Trace < Base
6
-
7
- required :uuid, :string, 1
8
- optional :endpoint, :string, 2
9
- repeated :spans, Span, 3
10
-
3
+ class Trace
11
4
  class Builder
12
5
  GC_CAT = 'noise.gc'.freeze
13
6
 
@@ -17,23 +10,23 @@ module Skylight
17
10
 
18
11
  def endpoint=(value)
19
12
  @endpoint = value.freeze
13
+ @native_builder.native_set_name(value)
20
14
  end
21
15
 
22
16
  def initialize(instrumenter, endpoint, start, cat, title=nil, desc=nil, annot=nil)
23
17
  raise ArgumentError, 'instrumenter is required' unless instrumenter
24
18
 
25
- @instrumenter = instrumenter
26
- @endpoint = endpoint.freeze
27
- @start = start
28
- @spans = []
29
- @stack = []
30
- @submitted = false
19
+ start = normalize_time(start)
31
20
 
32
- # Tracks the AS::N stack
33
- @notifications = []
21
+ @native_builder = ::Skylight::Trace.native_new(start, "TODO")
22
+ @native_builder.native_set_name(endpoint)
34
23
 
35
- # Track time
36
- @last_seen_time = start
24
+ @instrumenter = instrumenter
25
+ @endpoint = endpoint.freeze
26
+ @submitted = false
27
+ @start = start
28
+
29
+ @notifications = []
37
30
 
38
31
  if Hash === title
39
32
  annot = title
@@ -43,26 +36,41 @@ module Skylight
43
36
  desc = nil
44
37
  end
45
38
 
46
- # Create the root node
47
- @root = start(@start, cat, title, desc, annot)
48
- @gc = config.gc.track
39
+ # create the root node
40
+ @root = @native_builder.native_start_span(@start, cat)
41
+ @native_builder.native_span_set_title(@root, title) if title
42
+ @native_builder.native_span_set_description(@root, desc) if desc
43
+
44
+ @gc = config.gc.track unless ENV.key?("SKYLIGHT_DISABLE_GC_TRACKING")
45
+ end
46
+
47
+ def serialize
48
+ raise "Can only serialize once" if @serialized
49
+ @serialized = true
50
+ @native_builder.native_serialize
49
51
  end
50
52
 
51
53
  def config
52
54
  @instrumenter.config
53
55
  end
54
56
 
55
- def record(cat, *args)
56
- annot = args.pop if Hash === args
57
- title = args.shift
58
- desc = args.shift
59
- now = adjust_for_skew(Util::Clock.micros)
57
+ def record(cat, title=nil, desc=nil, annot=nil)
58
+ if Hash === title
59
+ annot = title
60
+ title = desc = nil
61
+ elsif Hash === desc
62
+ annot = desc
63
+ desc = nil
64
+ end
65
+
66
+ title.freeze
67
+ desc.freeze
60
68
 
61
69
  desc = @instrumenter.limited_description(desc)
62
70
 
63
- sp = span(now - gc_time, cat, title, desc, annot)
64
- inc_children
65
- @spans << sp.build(0)
71
+ time = Util::Clock.nanos - gc_time
72
+
73
+ stop(start(time, cat, title, desc), time)
66
74
 
67
75
  nil
68
76
  end
@@ -80,7 +88,7 @@ module Skylight
80
88
  desc.freeze
81
89
 
82
90
  original_desc = desc
83
- now = adjust_for_skew(Util::Clock.micros)
91
+ now = Util::Clock.nanos
84
92
  desc = @instrumenter.limited_description(desc)
85
93
 
86
94
  if desc == Instrumenter::TOO_MANY_UNIQUES
@@ -94,7 +102,7 @@ module Skylight
94
102
 
95
103
  def done(span)
96
104
  return unless span
97
- stop(span, adjust_for_skew(Util::Clock.micros) - gc_time)
105
+ stop(span, Util::Clock.nanos - gc_time)
98
106
  end
99
107
 
100
108
  def release
@@ -102,37 +110,27 @@ module Skylight
102
110
  @instrumenter.current_trace = nil
103
111
  end
104
112
 
105
- def submit
106
- return if @submitted
107
-
108
- release
109
- @submitted = true
110
-
111
- now = adjust_for_skew(Util::Clock.micros)
112
-
113
- # Pop everything that is left
114
- while sp = pop
115
- @spans << sp.build(relativize(now) - sp.started_at)
116
- end
117
-
113
+ def traced
118
114
  time = gc_time
115
+ now = Util::Clock.nanos
119
116
 
120
117
  if time > 0
121
118
  t { fmt "tracking GC time; duration=%d", time }
122
- noise = start(now - time, GC_CAT, nil, nil, {})
123
- stop(noise, now)
119
+ stop(start(now - time, GC_CAT, nil, nil, {}), now)
124
120
  end
125
121
 
126
- if sp = @stack.pop
127
- @spans << sp.build(relativize(now) - sp.started_at)
128
- end
122
+ stop(@root, now)
123
+ end
124
+
125
+ def submit
126
+ return if @submitted
127
+
128
+ release
129
+ @submitted = true
129
130
 
130
- t = Trace.new(
131
- uuid: 'TODO',
132
- endpoint: endpoint,
133
- spans: spans)
131
+ traced
134
132
 
135
- @instrumenter.process(t)
133
+ @instrumenter.process(@native_builder)
136
134
  rescue Exception => e
137
135
  error e
138
136
  t { e.backtrace.join("\n") }
@@ -140,63 +138,24 @@ module Skylight
140
138
 
141
139
  private
142
140
 
143
- def start(time, cat, title, desc, annot)
144
- sp = span(time, cat, title, desc, annot)
145
-
146
- push(sp)
147
-
148
- sp
141
+ def start(time, cat, title, desc, annot=nil)
142
+ span(normalize_time(time), cat, title, desc, annot)
149
143
  end
150
144
 
151
145
  def stop(span, time)
152
- until span == (sp = pop)
153
- return unless sp
154
- @spans << sp.build(relativize(time) - sp.started_at)
155
- end
156
-
157
- @spans << span.build(relativize(time) - sp.started_at)
158
-
146
+ @native_builder.native_stop_span(span, normalize_time(time))
159
147
  nil
160
148
  end
161
149
 
162
- def span(time, cat, title, desc, annot)
163
- Span::Builder.new(
164
- self, time, relativize(time),
165
- cat, title, desc, annot)
166
- end
167
-
168
- def pop
169
- return unless @stack.length > 1
170
- @stack.pop
171
- end
172
-
173
- def push(sp)
174
- inc_children
175
- @stack << sp
176
- end
177
-
178
- def inc_children
179
- return unless sp = @stack[-1]
180
- sp.children += 1
150
+ def normalize_time(time)
151
+ time.to_i / 100_000
181
152
  end
182
153
 
183
- def relativize(time)
184
- if parent = @stack[-1]
185
- ((time - parent.time) / 100).to_i
186
- else
187
- ((time - @start) / 100).to_i
188
- end
189
- end
190
-
191
- # Sadely, we don't have access to a pure monotonic clock in ruby, so we
192
- # need to cheat a little.
193
- def adjust_for_skew(time)
194
- if time <= @last_seen_time
195
- return @last_seen_time
196
- end
197
-
198
- @last_seen_time = time
199
- time
154
+ def span(time, cat, title=nil, desc=nil, annot=nil)
155
+ sp = @native_builder.native_start_span(time, cat.to_s)
156
+ @native_builder.native_span_set_title(sp, title.to_s) if title
157
+ @native_builder.native_span_set_description(sp, desc.to_s) if desc
158
+ sp
200
159
  end
201
160
 
202
161
  def gc_time