skylight 0.2.7 → 0.3.0.rc.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. checksums.yaml +4 -4
  2. data/ext/checksums.yml +3 -0
  3. data/ext/extconf.rb +90 -0
  4. data/ext/skylight.map +4 -0
  5. data/ext/skylight_native.c +613 -0
  6. data/lib/skylight.rb +50 -7
  7. data/lib/skylight/config.rb +7 -1
  8. data/lib/skylight/helpers.rb +2 -2
  9. data/lib/skylight/instrumenter.rb +9 -5
  10. data/lib/skylight/messages.rb +13 -13
  11. data/lib/skylight/messages/error.rb +10 -6
  12. data/lib/skylight/messages/hello.rb +4 -45
  13. data/lib/skylight/messages/trace.rb +62 -103
  14. data/lib/skylight/messages/trace_envelope.rb +19 -0
  15. data/lib/skylight/native.rb +80 -0
  16. data/lib/skylight/normalizers/process_action.rb +1 -3
  17. data/lib/skylight/probes.rb +1 -1
  18. data/lib/skylight/subscriber.rb +1 -1
  19. data/lib/skylight/util/clock.rb +13 -6
  20. data/lib/skylight/version.rb +1 -1
  21. data/lib/skylight/worker/builder.rb +1 -1
  22. data/lib/skylight/worker/collector.rb +20 -32
  23. data/lib/skylight/worker/connection.rb +2 -2
  24. data/lib/skylight/worker/embedded.rb +18 -0
  25. data/lib/skylight/worker/server.rb +3 -3
  26. data/lib/skylight/worker/standalone.rb +9 -10
  27. data/lib/sql_lexer.rb +0 -1
  28. data/lib/sql_lexer/lexer.rb +50 -4
  29. data/lib/sql_lexer/version.rb +1 -1
  30. metadata +19 -22
  31. data/lib/skylight/messages/annotation.rb +0 -13
  32. data/lib/skylight/messages/base.rb +0 -24
  33. data/lib/skylight/messages/batch.rb +0 -12
  34. data/lib/skylight/messages/endpoint.rb +0 -12
  35. data/lib/skylight/messages/event.rb +0 -12
  36. data/lib/skylight/messages/span.rb +0 -166
  37. data/lib/skylight/vendor/beefcake.rb +0 -292
  38. data/lib/skylight/vendor/beefcake/buffer.rb +0 -119
  39. data/lib/skylight/vendor/beefcake/decode.rb +0 -107
  40. data/lib/skylight/vendor/beefcake/encode.rb +0 -132
@@ -0,0 +1,19 @@
1
+ module Skylight
2
+ module Messages
3
+ class TraceEnvelope
4
+ def self.deserialize(data)
5
+ new(data)
6
+ end
7
+
8
+ attr_reader :data
9
+
10
+ def initialize(data)
11
+ @data = data
12
+ end
13
+
14
+ def endpoint_name
15
+ Skylight::Trace.native_name_from_serialized(@data)
16
+ end
17
+ end
18
+ end
19
+ end
@@ -0,0 +1,80 @@
1
+ module Skylight
2
+ class Hello
3
+ DIGITS = /^\s*\d+\s*$/
4
+
5
+ alias serialize native_serialize
6
+ alias version native_get_version
7
+
8
+ class << self
9
+ alias deserialize native_load
10
+ end
11
+
12
+ def cmd
13
+ native_cmd_length.times.map do |offset|
14
+ native_cmd_get(offset)
15
+ end
16
+ end
17
+
18
+ def newer?(other = VERSION)
19
+ other = split(other)
20
+ curr = split(version)
21
+
22
+ [other.length, curr.length].max.times do |i|
23
+ next if other[i] == curr[i]
24
+ return true unless other[i]
25
+
26
+ if other[i] =~ DIGITS
27
+ if curr[i] =~ DIGITS
28
+ other_i = other[i].to_i
29
+ curr_i = curr[i].to_i
30
+
31
+ next if other_i == curr_i
32
+
33
+ return curr_i > other_i
34
+ else
35
+ return false
36
+ end
37
+ else
38
+ if curr[i] =~ DIGITS
39
+ return true
40
+ else
41
+ next if curr[i] == other[i]
42
+ return curr[i] > other[i]
43
+ end
44
+ end
45
+ end
46
+
47
+ false
48
+ end
49
+
50
+ private
51
+
52
+ def split(v)
53
+ v.split('.')
54
+ end
55
+
56
+ end
57
+
58
+ class Error
59
+ alias serialize native_serialize
60
+ alias type native_get_group
61
+ alias description native_get_description
62
+ alias details native_get_details
63
+
64
+ class << self
65
+ alias deserialize native_load
66
+ end
67
+ end
68
+
69
+ class Trace
70
+ alias serialize native_serialize
71
+
72
+ class << self
73
+ alias deserialize native_load
74
+ end
75
+ end
76
+
77
+ class Batch
78
+ alias serialize native_serialize
79
+ end
80
+ end
@@ -21,9 +21,7 @@ module Skylight
21
21
  normalized = {}
22
22
 
23
23
  PAYLOAD_KEYS.each do |key|
24
- val = payload[key]
25
- val = val.inspect unless val.is_a?(String) || val.is_a?(Numeric)
26
- normalized[key] = val
24
+ normalized[key] = payload[key]
27
25
  end
28
26
 
29
27
  normalized
@@ -87,4 +87,4 @@ module ::Kernel
87
87
 
88
88
  ret
89
89
  end
90
- end
90
+ end
@@ -67,7 +67,7 @@ module Skylight
67
67
 
68
68
  while curr = trace.notifications.pop
69
69
  if curr.name == name
70
- curr.span.done if curr.span
70
+ trace.done(curr.span) if curr.span
71
71
  return
72
72
  end
73
73
  end
@@ -2,17 +2,24 @@ module Skylight
2
2
  module Util
3
3
  class Clock
4
4
 
5
- def micros
6
- n = Time.now
7
- n.to_i * 1_000_000 + n.usec
5
+ def absolute_secs
6
+ Time.now.to_i
7
+ end
8
+
9
+ def nanos
10
+ native_hrtime
8
11
  end
9
12
 
10
13
  def secs
11
- micros / 1_000_000
14
+ nanos / 1_000_000_000
15
+ end
16
+
17
+ def self.absolute_secs
18
+ default.absolute_secs
12
19
  end
13
20
 
14
- def self.micros
15
- default.micros
21
+ def self.nanos
22
+ default.nanos
16
23
  end
17
24
 
18
25
  def self.secs
@@ -1,4 +1,4 @@
1
1
  module Skylight
2
- VERSION = '0.2.7'
2
+ VERSION = '0.3.0.rc.3'
3
3
  end
4
4
 
@@ -19,7 +19,7 @@ module Skylight
19
19
  case s
20
20
  when 'embedded'
21
21
  trace "building embedded worker"
22
- Collector.new(config)
22
+ Embedded.new(Collector.new(config))
23
23
  when 'standalone'
24
24
  trace "building standalone worker"
25
25
 
@@ -7,7 +7,7 @@ module Skylight
7
7
 
8
8
  ENDPOINT = '/report'.freeze
9
9
  CONTENT_TYPE = 'content-type'.freeze
10
- SKYLIGHT_V1 = 'application/x-skylight-report-v1'.freeze
10
+ SKYLIGHT_V2 = 'application/x-skylight-report-v2'.freeze
11
11
 
12
12
  include Util::Logging
13
13
 
@@ -20,7 +20,6 @@ module Skylight
20
20
  @size = config[:'agent.sample']
21
21
  @batch = nil
22
22
  @interval = config[:'agent.interval']
23
- @buf = ""
24
23
  @refresh_at = 0
25
24
  @http_auth = Util::HTTP.new(config, :accounts)
26
25
  @http_report = nil
@@ -29,7 +28,7 @@ module Skylight
29
28
  t { fmt "starting collector; interval=%d; size=%d", @interval, @size }
30
29
  end
31
30
 
32
- def handle(msg, now = Util::Clock.secs)
31
+ def handle(msg, now = Util::Clock.absolute_secs)
33
32
  @batch ||= new_batch(now)
34
33
 
35
34
  if should_refresh_token?(now)
@@ -50,10 +49,10 @@ module Skylight
50
49
  return true unless msg
51
50
 
52
51
  case msg
53
- when Messages::Trace
52
+ when Messages::TraceEnvelope
54
53
  t { fmt "collector received trace" }
55
54
  @batch.push(msg)
56
- when Messages::Error
55
+ when Error
57
56
  send_error(msg)
58
57
  else
59
58
  debug "Received unknown message; class=%s", msg.class.to_s
@@ -96,7 +95,7 @@ module Skylight
96
95
  def finish
97
96
  t { fmt "collector finishing up" }
98
97
 
99
- now = Util::Clock.secs
98
+ now = Util::Clock.absolute_secs
100
99
 
101
100
  if should_refresh_token?(now)
102
101
  refresh_report_token(now)
@@ -114,8 +113,7 @@ module Skylight
114
113
 
115
114
  debug "flushing batch; size=%d", batch.sample.count
116
115
 
117
- @buf.clear
118
- @http_report.post(ENDPOINT, batch.encode(@buf), CONTENT_TYPE => SKYLIGHT_V1)
116
+ @http_report.post(ENDPOINT, batch.encode, CONTENT_TYPE => SKYLIGHT_V2)
119
117
  end
120
118
 
121
119
  def refresh_report_token(now)
@@ -137,13 +135,15 @@ module Skylight
137
135
  tok = res.body['session']
138
136
  tok = tok['token'] if tok
139
137
 
138
+ t { "New token: #{tok}" }
139
+
140
140
  if tok
141
- @refresh_at = now + 30
141
+ @refresh_at = now + 1800 # 30 minutes
142
142
  @http_report = Util::HTTP.new(config, :report)
143
143
  @http_report.authentication = tok
144
144
  else
145
145
  if @http_report
146
- @refresh_at = now + 30
146
+ @refresh_at = now + 60
147
147
  end
148
148
  warn "server did not return a session token"
149
149
  end
@@ -157,6 +157,7 @@ module Skylight
157
157
  end
158
158
 
159
159
  def has_report_token?(now)
160
+ return true if config.ignore_token?
160
161
  return unless @http_report
161
162
  now < @refresh_at + (3600 * 3 - 660)
162
163
  end
@@ -184,6 +185,7 @@ module Skylight
184
185
 
185
186
  def should_flush?(now)
186
187
  return true if @config.constant_flush?
188
+ t { "Checking if should flush at #{now} (@flush_at is #{@flush_at})" }
187
189
  now >= @flush_at
188
190
  end
189
191
 
@@ -193,37 +195,23 @@ module Skylight
193
195
 
194
196
  def push(trace)
195
197
  # Count it
196
- @counts[trace.endpoint] += 1
198
+ @counts[trace.endpoint_name] += 1
197
199
  # Push the trace into the sample
198
200
  @sample << trace
199
201
  end
200
202
 
201
- def encode(buf)
202
- endpoints = {}
203
+ def encode
204
+ batch = Skylight::Batch.native_new(from, config[:hostname])
203
205
 
204
206
  sample.each do |trace|
205
- unless name = trace.endpoint
206
- debug "trace missing name -- dropping"
207
- next
208
- end
209
-
210
- trace.endpoint = nil
211
-
212
- ep = (endpoints[name] ||= Messages::Endpoint.new(
213
- name: name, traces: [], count: @counts[name]))
214
-
215
- ep.traces << trace
207
+ batch.native_move_in(trace.data)
216
208
  end
217
209
 
218
- t { fmt "encoding batch; endpoints=%p", endpoints.keys }
219
-
220
- Messages::Batch.new(
221
- timestamp: from,
222
- hostname: config[:hostname],
223
- endpoints: endpoints.values).
224
- encode(buf)
210
+ @counts.each do |endpoint_name,count|
211
+ batch.native_set_endpoint_count(endpoint_name, count)
212
+ end
225
213
 
226
- buf
214
+ batch.native_serialize
227
215
  end
228
216
  end
229
217
 
@@ -44,7 +44,7 @@ module Skylight
44
44
  def maybe_read_message
45
45
  if @len && @buf.bytesize >= @len + FRAME_HDR_LEN
46
46
  mid = read_message_id
47
- klass = Messages.get(mid)
47
+ klass = Messages::ID_TO_KLASS.fetch(mid)
48
48
  data = @buf[FRAME_HDR_LEN, @len]
49
49
  @buf = @buf[(FRAME_HDR_LEN + @len)..-1] || ""
50
50
 
@@ -59,7 +59,7 @@ module Skylight
59
59
  end
60
60
 
61
61
  begin
62
- return klass.decode(data)
62
+ return klass.deserialize(data)
63
63
  rescue Exception => e
64
64
  # reraise protobuf decoding exceptions
65
65
  raise IpcProtoError, e.message
@@ -1,6 +1,24 @@
1
1
  module Skylight
2
2
  module Worker
3
3
  class Embedded
4
+ def initialize(collector)
5
+ @collector = collector
6
+ end
7
+
8
+ def spawn
9
+ @collector.spawn
10
+ end
11
+
12
+ def shutdown
13
+ @collector.shutdown
14
+ end
15
+
16
+ def submit(msg)
17
+ decoder = Messages::ID_TO_KLASS.fetch(Messages::KLASS_TO_ID.fetch(msg.class))
18
+ msg = decoder.deserialize(msg.serialize)
19
+
20
+ @collector.submit(msg)
21
+ end
4
22
  end
5
23
  end
6
24
  end
@@ -221,12 +221,12 @@ module Skylight
221
221
  case msg
222
222
  when nil
223
223
  return
224
- when Messages::Hello
224
+ when Hello
225
225
  if msg.newer?
226
226
  info "newer version of agent deployed - restarting; curr=%s; new=%s", VERSION, msg.version
227
227
  reload(msg)
228
228
  end
229
- when Messages::Base
229
+ when Messages::TraceEnvelope, Error
230
230
  t { "received message" }
231
231
  @collector.submit(msg)
232
232
  when :unknown
@@ -327,7 +327,7 @@ module Skylight
327
327
 
328
328
  def get_memory_usage
329
329
  `ps -o rss= -p #{Process.pid}`.to_i / 1024
330
- rescue Errno::ENOENT, Errno::EINTR
330
+ rescue Errno::ENOENT
331
331
  0
332
332
  end
333
333
  end
@@ -63,7 +63,7 @@ module Skylight
63
63
  end
64
64
 
65
65
  def submit(msg)
66
- unless msg.respond_to?(:encode)
66
+ unless msg.respond_to?(:encode) || msg.respond_to?(:native_serialize)
67
67
  raise ArgumentError, "message not encodable"
68
68
  end
69
69
 
@@ -217,11 +217,13 @@ module Skylight
217
217
  end
218
218
 
219
219
  def write_msg(sock, msg)
220
- buf = msg.encode.to_s
221
- frame = [ msg.message_id, buf.bytesize ].pack("LL")
220
+ t { "writing a #{msg.class} on the wire" }
221
+ id = Messages::KLASS_TO_ID.fetch(msg.class)
222
+ buf = msg.serialize
222
223
 
223
- write(sock, frame) &&
224
- write(sock, buf)
224
+ frame = [ id, buf.bytesize ].pack("LL")
225
+
226
+ write(sock, frame) && write(sock, buf)
225
227
  end
226
228
 
227
229
  SOCK_TIMEOUT_VAL = [ 0, 0.01 * 1_000_000 ].pack("l_2")
@@ -354,11 +356,8 @@ module Skylight
354
356
  end
355
357
  end
356
358
 
357
- def build_hello(config_version = nil)
358
- Messages::Hello.new(
359
- version: VERSION,
360
- cmd: SUBPROCESS_CMD,
361
- config: config_version)
359
+ def build_hello
360
+ Messages::Hello.build(VERSION, SUBPROCESS_CMD)
362
361
  end
363
362
 
364
363
  def build_queue
data/lib/sql_lexer.rb CHANGED
@@ -2,5 +2,4 @@ require "sql_lexer/version"
2
2
  require "sql_lexer/lexer"
3
3
 
4
4
  module SqlLexer
5
- # Your code goes here...
6
5
  end
@@ -29,6 +29,8 @@ module SqlLexer
29
29
  StartString = %Q<'>
30
30
  StartDigit = %q<[\p{Digit}\.]>
31
31
 
32
+ StartSelect = %Q<SELECT(?=(?:[#{WS}]|#{OpPart}))>
33
+
32
34
  # Binds that are also IDs do not need to be included here, since AfterOp (which uses StartBind)
33
35
  # also checks for StartAnyId
34
36
  StartBind = %Q<#{StartString}|#{StartDigit}|#{SpecialOps}>
@@ -85,7 +87,9 @@ module SqlLexer
85
87
  TkArray = %r<#{ArrayOp}>iu
86
88
  TkArrayIndex = %r<#{ArrayIndexOp}>iu
87
89
  TkSpecialOp = %r<#{SpecialOps}>iu
88
- TkStartSelect = %r<SELECT(?=(?:[#{WS}]|#{OpPart}))>iu
90
+ TkStartSelect = %r<#{StartSelect}>iu
91
+ TkStartSubquery = %r<\(#{OptWS}#{StartSelect}>iu
92
+ TkCloseParen = %r<#{OptWS}\)>u
89
93
 
90
94
  STATE_HANDLERS = {
91
95
  begin: :process_begin,
@@ -97,6 +101,7 @@ module SqlLexer
97
101
  table_name: :process_table_name,
98
102
  end: :process_end,
99
103
  special: :process_special,
104
+ subquery: :process_subquery,
100
105
  in: :process_in,
101
106
  array: :process_array
102
107
  }
@@ -351,8 +356,12 @@ module SqlLexer
351
356
  def process_special
352
357
  if @scanner.skip(TkIn)
353
358
  @scanner.skip(TkOptWS)
354
- @scanner.skip(/\(/u)
355
- @state = :in
359
+ if @scanner.skip(TkStartSubquery)
360
+ @state = :subquery
361
+ else
362
+ @scanner.skip(/\(/u)
363
+ @state = :in
364
+ end
356
365
  elsif @scanner.skip(TkArray)
357
366
  @scanner.skip(/\[/u)
358
367
  @state = :array
@@ -362,11 +371,48 @@ module SqlLexer
362
371
  else
363
372
  @state = :end
364
373
  end
374
+ elsif @scanner.skip(TkStartSubquery)
375
+ @state = :subquery
365
376
  elsif @scanner.skip(TkArrayIndex)
366
377
  @state = :tokens
367
378
  end
368
379
  end
369
380
 
381
+ def process_subquery
382
+ nest = 1
383
+ iterations = 0
384
+
385
+ while nest > 0
386
+ iterations += 1
387
+
388
+ if iterations > 10_000
389
+ raise "The SQL '#{@scanner.string}' could not be parsed because of too many iterations in subquery"
390
+ end
391
+
392
+ if @debug
393
+ p @state
394
+ p @scanner
395
+ p nest
396
+ p @scanner.peek(1)
397
+ end
398
+
399
+ if @scanner.skip(TkStartSubquery)
400
+ nest += 1
401
+ @state = :tokens
402
+ elsif @scanner.skip(TkCloseParen)
403
+ nest -= 1
404
+ break if nest.zero?
405
+ @state = :tokens
406
+ elsif @state == :subquery
407
+ @state = :tokens
408
+ end
409
+
410
+ __send__ STATE_HANDLERS[@state]
411
+ end
412
+
413
+ @state = :tokens
414
+ end
415
+
370
416
  def process_in
371
417
  nest = 1
372
418
  iterations = 0
@@ -390,7 +436,7 @@ module SqlLexer
390
436
  if @scanner.skip(/\(/u)
391
437
  nest += 1
392
438
  process_tokens
393
- elsif @scanner.skip(/\)/u)
439
+ elsif @scanner.skip(TkCloseParen)
394
440
  nest -= 1
395
441
  break if nest.zero?
396
442
  process_tokens