skylight 0.2.7 → 0.3.0.rc.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/ext/checksums.yml +3 -0
- data/ext/extconf.rb +90 -0
- data/ext/skylight.map +4 -0
- data/ext/skylight_native.c +613 -0
- data/lib/skylight.rb +50 -7
- data/lib/skylight/config.rb +7 -1
- data/lib/skylight/helpers.rb +2 -2
- data/lib/skylight/instrumenter.rb +9 -5
- data/lib/skylight/messages.rb +13 -13
- data/lib/skylight/messages/error.rb +10 -6
- data/lib/skylight/messages/hello.rb +4 -45
- data/lib/skylight/messages/trace.rb +62 -103
- data/lib/skylight/messages/trace_envelope.rb +19 -0
- data/lib/skylight/native.rb +80 -0
- data/lib/skylight/normalizers/process_action.rb +1 -3
- data/lib/skylight/probes.rb +1 -1
- data/lib/skylight/subscriber.rb +1 -1
- data/lib/skylight/util/clock.rb +13 -6
- data/lib/skylight/version.rb +1 -1
- data/lib/skylight/worker/builder.rb +1 -1
- data/lib/skylight/worker/collector.rb +20 -32
- data/lib/skylight/worker/connection.rb +2 -2
- data/lib/skylight/worker/embedded.rb +18 -0
- data/lib/skylight/worker/server.rb +3 -3
- data/lib/skylight/worker/standalone.rb +9 -10
- data/lib/sql_lexer.rb +0 -1
- data/lib/sql_lexer/lexer.rb +50 -4
- data/lib/sql_lexer/version.rb +1 -1
- metadata +19 -22
- data/lib/skylight/messages/annotation.rb +0 -13
- data/lib/skylight/messages/base.rb +0 -24
- data/lib/skylight/messages/batch.rb +0 -12
- data/lib/skylight/messages/endpoint.rb +0 -12
- data/lib/skylight/messages/event.rb +0 -12
- data/lib/skylight/messages/span.rb +0 -166
- data/lib/skylight/vendor/beefcake.rb +0 -292
- data/lib/skylight/vendor/beefcake/buffer.rb +0 -119
- data/lib/skylight/vendor/beefcake/decode.rb +0 -107
- data/lib/skylight/vendor/beefcake/encode.rb +0 -132
@@ -0,0 +1,19 @@
|
|
1
|
+
module Skylight
|
2
|
+
module Messages
|
3
|
+
class TraceEnvelope
|
4
|
+
def self.deserialize(data)
|
5
|
+
new(data)
|
6
|
+
end
|
7
|
+
|
8
|
+
attr_reader :data
|
9
|
+
|
10
|
+
def initialize(data)
|
11
|
+
@data = data
|
12
|
+
end
|
13
|
+
|
14
|
+
def endpoint_name
|
15
|
+
Skylight::Trace.native_name_from_serialized(@data)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,80 @@
|
|
1
|
+
module Skylight
|
2
|
+
class Hello
|
3
|
+
DIGITS = /^\s*\d+\s*$/
|
4
|
+
|
5
|
+
alias serialize native_serialize
|
6
|
+
alias version native_get_version
|
7
|
+
|
8
|
+
class << self
|
9
|
+
alias deserialize native_load
|
10
|
+
end
|
11
|
+
|
12
|
+
def cmd
|
13
|
+
native_cmd_length.times.map do |offset|
|
14
|
+
native_cmd_get(offset)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def newer?(other = VERSION)
|
19
|
+
other = split(other)
|
20
|
+
curr = split(version)
|
21
|
+
|
22
|
+
[other.length, curr.length].max.times do |i|
|
23
|
+
next if other[i] == curr[i]
|
24
|
+
return true unless other[i]
|
25
|
+
|
26
|
+
if other[i] =~ DIGITS
|
27
|
+
if curr[i] =~ DIGITS
|
28
|
+
other_i = other[i].to_i
|
29
|
+
curr_i = curr[i].to_i
|
30
|
+
|
31
|
+
next if other_i == curr_i
|
32
|
+
|
33
|
+
return curr_i > other_i
|
34
|
+
else
|
35
|
+
return false
|
36
|
+
end
|
37
|
+
else
|
38
|
+
if curr[i] =~ DIGITS
|
39
|
+
return true
|
40
|
+
else
|
41
|
+
next if curr[i] == other[i]
|
42
|
+
return curr[i] > other[i]
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
false
|
48
|
+
end
|
49
|
+
|
50
|
+
private
|
51
|
+
|
52
|
+
def split(v)
|
53
|
+
v.split('.')
|
54
|
+
end
|
55
|
+
|
56
|
+
end
|
57
|
+
|
58
|
+
class Error
|
59
|
+
alias serialize native_serialize
|
60
|
+
alias type native_get_group
|
61
|
+
alias description native_get_description
|
62
|
+
alias details native_get_details
|
63
|
+
|
64
|
+
class << self
|
65
|
+
alias deserialize native_load
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
class Trace
|
70
|
+
alias serialize native_serialize
|
71
|
+
|
72
|
+
class << self
|
73
|
+
alias deserialize native_load
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
class Batch
|
78
|
+
alias serialize native_serialize
|
79
|
+
end
|
80
|
+
end
|
data/lib/skylight/probes.rb
CHANGED
data/lib/skylight/subscriber.rb
CHANGED
data/lib/skylight/util/clock.rb
CHANGED
@@ -2,17 +2,24 @@ module Skylight
|
|
2
2
|
module Util
|
3
3
|
class Clock
|
4
4
|
|
5
|
-
def
|
6
|
-
|
7
|
-
|
5
|
+
def absolute_secs
|
6
|
+
Time.now.to_i
|
7
|
+
end
|
8
|
+
|
9
|
+
def nanos
|
10
|
+
native_hrtime
|
8
11
|
end
|
9
12
|
|
10
13
|
def secs
|
11
|
-
|
14
|
+
nanos / 1_000_000_000
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.absolute_secs
|
18
|
+
default.absolute_secs
|
12
19
|
end
|
13
20
|
|
14
|
-
def self.
|
15
|
-
default.
|
21
|
+
def self.nanos
|
22
|
+
default.nanos
|
16
23
|
end
|
17
24
|
|
18
25
|
def self.secs
|
data/lib/skylight/version.rb
CHANGED
@@ -7,7 +7,7 @@ module Skylight
|
|
7
7
|
|
8
8
|
ENDPOINT = '/report'.freeze
|
9
9
|
CONTENT_TYPE = 'content-type'.freeze
|
10
|
-
|
10
|
+
SKYLIGHT_V2 = 'application/x-skylight-report-v2'.freeze
|
11
11
|
|
12
12
|
include Util::Logging
|
13
13
|
|
@@ -20,7 +20,6 @@ module Skylight
|
|
20
20
|
@size = config[:'agent.sample']
|
21
21
|
@batch = nil
|
22
22
|
@interval = config[:'agent.interval']
|
23
|
-
@buf = ""
|
24
23
|
@refresh_at = 0
|
25
24
|
@http_auth = Util::HTTP.new(config, :accounts)
|
26
25
|
@http_report = nil
|
@@ -29,7 +28,7 @@ module Skylight
|
|
29
28
|
t { fmt "starting collector; interval=%d; size=%d", @interval, @size }
|
30
29
|
end
|
31
30
|
|
32
|
-
def handle(msg, now = Util::Clock.
|
31
|
+
def handle(msg, now = Util::Clock.absolute_secs)
|
33
32
|
@batch ||= new_batch(now)
|
34
33
|
|
35
34
|
if should_refresh_token?(now)
|
@@ -50,10 +49,10 @@ module Skylight
|
|
50
49
|
return true unless msg
|
51
50
|
|
52
51
|
case msg
|
53
|
-
when Messages::
|
52
|
+
when Messages::TraceEnvelope
|
54
53
|
t { fmt "collector received trace" }
|
55
54
|
@batch.push(msg)
|
56
|
-
when
|
55
|
+
when Error
|
57
56
|
send_error(msg)
|
58
57
|
else
|
59
58
|
debug "Received unknown message; class=%s", msg.class.to_s
|
@@ -96,7 +95,7 @@ module Skylight
|
|
96
95
|
def finish
|
97
96
|
t { fmt "collector finishing up" }
|
98
97
|
|
99
|
-
now = Util::Clock.
|
98
|
+
now = Util::Clock.absolute_secs
|
100
99
|
|
101
100
|
if should_refresh_token?(now)
|
102
101
|
refresh_report_token(now)
|
@@ -114,8 +113,7 @@ module Skylight
|
|
114
113
|
|
115
114
|
debug "flushing batch; size=%d", batch.sample.count
|
116
115
|
|
117
|
-
@
|
118
|
-
@http_report.post(ENDPOINT, batch.encode(@buf), CONTENT_TYPE => SKYLIGHT_V1)
|
116
|
+
@http_report.post(ENDPOINT, batch.encode, CONTENT_TYPE => SKYLIGHT_V2)
|
119
117
|
end
|
120
118
|
|
121
119
|
def refresh_report_token(now)
|
@@ -137,13 +135,15 @@ module Skylight
|
|
137
135
|
tok = res.body['session']
|
138
136
|
tok = tok['token'] if tok
|
139
137
|
|
138
|
+
t { "New token: #{tok}" }
|
139
|
+
|
140
140
|
if tok
|
141
|
-
@refresh_at = now + 30
|
141
|
+
@refresh_at = now + 1800 # 30 minutes
|
142
142
|
@http_report = Util::HTTP.new(config, :report)
|
143
143
|
@http_report.authentication = tok
|
144
144
|
else
|
145
145
|
if @http_report
|
146
|
-
@refresh_at = now +
|
146
|
+
@refresh_at = now + 60
|
147
147
|
end
|
148
148
|
warn "server did not return a session token"
|
149
149
|
end
|
@@ -157,6 +157,7 @@ module Skylight
|
|
157
157
|
end
|
158
158
|
|
159
159
|
def has_report_token?(now)
|
160
|
+
return true if config.ignore_token?
|
160
161
|
return unless @http_report
|
161
162
|
now < @refresh_at + (3600 * 3 - 660)
|
162
163
|
end
|
@@ -184,6 +185,7 @@ module Skylight
|
|
184
185
|
|
185
186
|
def should_flush?(now)
|
186
187
|
return true if @config.constant_flush?
|
188
|
+
t { "Checking if should flush at #{now} (@flush_at is #{@flush_at})" }
|
187
189
|
now >= @flush_at
|
188
190
|
end
|
189
191
|
|
@@ -193,37 +195,23 @@ module Skylight
|
|
193
195
|
|
194
196
|
def push(trace)
|
195
197
|
# Count it
|
196
|
-
@counts[trace.
|
198
|
+
@counts[trace.endpoint_name] += 1
|
197
199
|
# Push the trace into the sample
|
198
200
|
@sample << trace
|
199
201
|
end
|
200
202
|
|
201
|
-
def encode
|
202
|
-
|
203
|
+
def encode
|
204
|
+
batch = Skylight::Batch.native_new(from, config[:hostname])
|
203
205
|
|
204
206
|
sample.each do |trace|
|
205
|
-
|
206
|
-
debug "trace missing name -- dropping"
|
207
|
-
next
|
208
|
-
end
|
209
|
-
|
210
|
-
trace.endpoint = nil
|
211
|
-
|
212
|
-
ep = (endpoints[name] ||= Messages::Endpoint.new(
|
213
|
-
name: name, traces: [], count: @counts[name]))
|
214
|
-
|
215
|
-
ep.traces << trace
|
207
|
+
batch.native_move_in(trace.data)
|
216
208
|
end
|
217
209
|
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
timestamp: from,
|
222
|
-
hostname: config[:hostname],
|
223
|
-
endpoints: endpoints.values).
|
224
|
-
encode(buf)
|
210
|
+
@counts.each do |endpoint_name,count|
|
211
|
+
batch.native_set_endpoint_count(endpoint_name, count)
|
212
|
+
end
|
225
213
|
|
226
|
-
|
214
|
+
batch.native_serialize
|
227
215
|
end
|
228
216
|
end
|
229
217
|
|
@@ -44,7 +44,7 @@ module Skylight
|
|
44
44
|
def maybe_read_message
|
45
45
|
if @len && @buf.bytesize >= @len + FRAME_HDR_LEN
|
46
46
|
mid = read_message_id
|
47
|
-
klass = Messages.
|
47
|
+
klass = Messages::ID_TO_KLASS.fetch(mid)
|
48
48
|
data = @buf[FRAME_HDR_LEN, @len]
|
49
49
|
@buf = @buf[(FRAME_HDR_LEN + @len)..-1] || ""
|
50
50
|
|
@@ -59,7 +59,7 @@ module Skylight
|
|
59
59
|
end
|
60
60
|
|
61
61
|
begin
|
62
|
-
return klass.
|
62
|
+
return klass.deserialize(data)
|
63
63
|
rescue Exception => e
|
64
64
|
# reraise protobuf decoding exceptions
|
65
65
|
raise IpcProtoError, e.message
|
@@ -1,6 +1,24 @@
|
|
1
1
|
module Skylight
|
2
2
|
module Worker
|
3
3
|
class Embedded
|
4
|
+
def initialize(collector)
|
5
|
+
@collector = collector
|
6
|
+
end
|
7
|
+
|
8
|
+
def spawn
|
9
|
+
@collector.spawn
|
10
|
+
end
|
11
|
+
|
12
|
+
def shutdown
|
13
|
+
@collector.shutdown
|
14
|
+
end
|
15
|
+
|
16
|
+
def submit(msg)
|
17
|
+
decoder = Messages::ID_TO_KLASS.fetch(Messages::KLASS_TO_ID.fetch(msg.class))
|
18
|
+
msg = decoder.deserialize(msg.serialize)
|
19
|
+
|
20
|
+
@collector.submit(msg)
|
21
|
+
end
|
4
22
|
end
|
5
23
|
end
|
6
24
|
end
|
@@ -221,12 +221,12 @@ module Skylight
|
|
221
221
|
case msg
|
222
222
|
when nil
|
223
223
|
return
|
224
|
-
when
|
224
|
+
when Hello
|
225
225
|
if msg.newer?
|
226
226
|
info "newer version of agent deployed - restarting; curr=%s; new=%s", VERSION, msg.version
|
227
227
|
reload(msg)
|
228
228
|
end
|
229
|
-
when Messages::
|
229
|
+
when Messages::TraceEnvelope, Error
|
230
230
|
t { "received message" }
|
231
231
|
@collector.submit(msg)
|
232
232
|
when :unknown
|
@@ -327,7 +327,7 @@ module Skylight
|
|
327
327
|
|
328
328
|
def get_memory_usage
|
329
329
|
`ps -o rss= -p #{Process.pid}`.to_i / 1024
|
330
|
-
rescue Errno::ENOENT
|
330
|
+
rescue Errno::ENOENT
|
331
331
|
0
|
332
332
|
end
|
333
333
|
end
|
@@ -63,7 +63,7 @@ module Skylight
|
|
63
63
|
end
|
64
64
|
|
65
65
|
def submit(msg)
|
66
|
-
unless msg.respond_to?(:encode)
|
66
|
+
unless msg.respond_to?(:encode) || msg.respond_to?(:native_serialize)
|
67
67
|
raise ArgumentError, "message not encodable"
|
68
68
|
end
|
69
69
|
|
@@ -217,11 +217,13 @@ module Skylight
|
|
217
217
|
end
|
218
218
|
|
219
219
|
def write_msg(sock, msg)
|
220
|
-
|
221
|
-
|
220
|
+
t { "writing a #{msg.class} on the wire" }
|
221
|
+
id = Messages::KLASS_TO_ID.fetch(msg.class)
|
222
|
+
buf = msg.serialize
|
222
223
|
|
223
|
-
|
224
|
-
|
224
|
+
frame = [ id, buf.bytesize ].pack("LL")
|
225
|
+
|
226
|
+
write(sock, frame) && write(sock, buf)
|
225
227
|
end
|
226
228
|
|
227
229
|
SOCK_TIMEOUT_VAL = [ 0, 0.01 * 1_000_000 ].pack("l_2")
|
@@ -354,11 +356,8 @@ module Skylight
|
|
354
356
|
end
|
355
357
|
end
|
356
358
|
|
357
|
-
def build_hello
|
358
|
-
Messages::Hello.
|
359
|
-
version: VERSION,
|
360
|
-
cmd: SUBPROCESS_CMD,
|
361
|
-
config: config_version)
|
359
|
+
def build_hello
|
360
|
+
Messages::Hello.build(VERSION, SUBPROCESS_CMD)
|
362
361
|
end
|
363
362
|
|
364
363
|
def build_queue
|
data/lib/sql_lexer.rb
CHANGED
data/lib/sql_lexer/lexer.rb
CHANGED
@@ -29,6 +29,8 @@ module SqlLexer
|
|
29
29
|
StartString = %Q<'>
|
30
30
|
StartDigit = %q<[\p{Digit}\.]>
|
31
31
|
|
32
|
+
StartSelect = %Q<SELECT(?=(?:[#{WS}]|#{OpPart}))>
|
33
|
+
|
32
34
|
# Binds that are also IDs do not need to be included here, since AfterOp (which uses StartBind)
|
33
35
|
# also checks for StartAnyId
|
34
36
|
StartBind = %Q<#{StartString}|#{StartDigit}|#{SpecialOps}>
|
@@ -85,7 +87,9 @@ module SqlLexer
|
|
85
87
|
TkArray = %r<#{ArrayOp}>iu
|
86
88
|
TkArrayIndex = %r<#{ArrayIndexOp}>iu
|
87
89
|
TkSpecialOp = %r<#{SpecialOps}>iu
|
88
|
-
TkStartSelect = %r
|
90
|
+
TkStartSelect = %r<#{StartSelect}>iu
|
91
|
+
TkStartSubquery = %r<\(#{OptWS}#{StartSelect}>iu
|
92
|
+
TkCloseParen = %r<#{OptWS}\)>u
|
89
93
|
|
90
94
|
STATE_HANDLERS = {
|
91
95
|
begin: :process_begin,
|
@@ -97,6 +101,7 @@ module SqlLexer
|
|
97
101
|
table_name: :process_table_name,
|
98
102
|
end: :process_end,
|
99
103
|
special: :process_special,
|
104
|
+
subquery: :process_subquery,
|
100
105
|
in: :process_in,
|
101
106
|
array: :process_array
|
102
107
|
}
|
@@ -351,8 +356,12 @@ module SqlLexer
|
|
351
356
|
def process_special
|
352
357
|
if @scanner.skip(TkIn)
|
353
358
|
@scanner.skip(TkOptWS)
|
354
|
-
@scanner.skip(
|
355
|
-
|
359
|
+
if @scanner.skip(TkStartSubquery)
|
360
|
+
@state = :subquery
|
361
|
+
else
|
362
|
+
@scanner.skip(/\(/u)
|
363
|
+
@state = :in
|
364
|
+
end
|
356
365
|
elsif @scanner.skip(TkArray)
|
357
366
|
@scanner.skip(/\[/u)
|
358
367
|
@state = :array
|
@@ -362,11 +371,48 @@ module SqlLexer
|
|
362
371
|
else
|
363
372
|
@state = :end
|
364
373
|
end
|
374
|
+
elsif @scanner.skip(TkStartSubquery)
|
375
|
+
@state = :subquery
|
365
376
|
elsif @scanner.skip(TkArrayIndex)
|
366
377
|
@state = :tokens
|
367
378
|
end
|
368
379
|
end
|
369
380
|
|
381
|
+
def process_subquery
|
382
|
+
nest = 1
|
383
|
+
iterations = 0
|
384
|
+
|
385
|
+
while nest > 0
|
386
|
+
iterations += 1
|
387
|
+
|
388
|
+
if iterations > 10_000
|
389
|
+
raise "The SQL '#{@scanner.string}' could not be parsed because of too many iterations in subquery"
|
390
|
+
end
|
391
|
+
|
392
|
+
if @debug
|
393
|
+
p @state
|
394
|
+
p @scanner
|
395
|
+
p nest
|
396
|
+
p @scanner.peek(1)
|
397
|
+
end
|
398
|
+
|
399
|
+
if @scanner.skip(TkStartSubquery)
|
400
|
+
nest += 1
|
401
|
+
@state = :tokens
|
402
|
+
elsif @scanner.skip(TkCloseParen)
|
403
|
+
nest -= 1
|
404
|
+
break if nest.zero?
|
405
|
+
@state = :tokens
|
406
|
+
elsif @state == :subquery
|
407
|
+
@state = :tokens
|
408
|
+
end
|
409
|
+
|
410
|
+
__send__ STATE_HANDLERS[@state]
|
411
|
+
end
|
412
|
+
|
413
|
+
@state = :tokens
|
414
|
+
end
|
415
|
+
|
370
416
|
def process_in
|
371
417
|
nest = 1
|
372
418
|
iterations = 0
|
@@ -390,7 +436,7 @@ module SqlLexer
|
|
390
436
|
if @scanner.skip(/\(/u)
|
391
437
|
nest += 1
|
392
438
|
process_tokens
|
393
|
-
elsif @scanner.skip(
|
439
|
+
elsif @scanner.skip(TkCloseParen)
|
394
440
|
nest -= 1
|
395
441
|
break if nest.zero?
|
396
442
|
process_tokens
|