skylight 0.2.7 → 0.3.0.rc.3

data/lib/skylight/messages/trace_envelope.rb

@@ -0,0 +1,19 @@
+module Skylight
+  module Messages
+    class TraceEnvelope
+      def self.deserialize(data)
+        new(data)
+      end
+
+      attr_reader :data
+
+      def initialize(data)
+        @data = data
+      end
+
+      def endpoint_name
+        Skylight::Trace.native_name_from_serialized(@data)
+      end
+    end
+  end
+end

data/lib/skylight/native.rb

@@ -0,0 +1,80 @@
+module Skylight
+  class Hello
+    DIGITS = /^\s*\d+\s*$/
+
+    alias serialize native_serialize
+    alias version native_get_version
+
+    class << self
+      alias deserialize native_load
+    end
+
+    def cmd
+      native_cmd_length.times.map do |offset|
+        native_cmd_get(offset)
+      end
+    end
+
+    def newer?(other = VERSION)
+      other = split(other)
+      curr  = split(version)
+
+      [other.length, curr.length].max.times do |i|
+        next if other[i] == curr[i]
+        return true unless other[i]
+
+        if other[i] =~ DIGITS
+          if curr[i] =~ DIGITS
+            other_i = other[i].to_i
+            curr_i = curr[i].to_i
+
+            next if other_i == curr_i
+
+            return curr_i > other_i
+          else
+            return false
+          end
+        else
+          if curr[i] =~ DIGITS
+            return true
+          else
+            next if curr[i] == other[i]
+            return curr[i] > other[i]
+          end
+        end
+      end
+
+      false
+    end
+
+    private
+
+      def split(v)
+        v.split('.')
+      end
+
+  end
+
+  class Error
+    alias serialize native_serialize
+    alias type native_get_group
+    alias description native_get_description
+    alias details native_get_details
+
+    class << self
+      alias deserialize native_load
+    end
+  end
+
+  class Trace
+    alias serialize native_serialize
+
+    class << self
+      alias deserialize native_load
+    end
+  end
+
+  class Batch
+    alias serialize native_serialize
+  end
+end

data/lib/skylight/normalizers/process_action.rb

@@ -21,9 +21,7 @@ module Skylight
         normalized = {}
 
         PAYLOAD_KEYS.each do |key|
-          val = payload[key]
-          val = val.inspect unless val.is_a?(String) || val.is_a?(Numeric)
-          normalized[key] = val
+          normalized[key] = payload[key]
         end
 
         normalized

data/lib/skylight/probes.rb

@@ -87,4 +87,4 @@ module ::Kernel
 
     ret
   end
-end
+end

data/lib/skylight/subscriber.rb

@@ -67,7 +67,7 @@ module Skylight
 
       while curr = trace.notifications.pop
         if curr.name == name
-          curr.span.done if curr.span
+          trace.done(curr.span) if curr.span
           return
         end
       end

data/lib/skylight/util/clock.rb

@@ -2,17 +2,24 @@ module Skylight
   module Util
     class Clock
 
-      def micros
-        n = Time.now
-        n.to_i * 1_000_000 + n.usec
+      def absolute_secs
+        Time.now.to_i
+      end
+
+      def nanos
+        native_hrtime
       end
 
       def secs
-        micros / 1_000_000
+        nanos / 1_000_000_000
+      end
+
+      def self.absolute_secs
+        default.absolute_secs
       end
 
-      def self.micros
-        default.micros
+      def self.nanos
+        default.nanos
       end
 
       def self.secs

data/lib/skylight/version.rb

@@ -1,4 +1,4 @@
 module Skylight
-  VERSION = '0.2.7'
+  VERSION = '0.3.0.rc.3'
 end
 

data/lib/skylight/worker/builder.rb

@@ -19,7 +19,7 @@ module Skylight
         case s
         when 'embedded'
           trace "building embedded worker"
-          Collector.new(config)
+          Embedded.new(Collector.new(config))
         when 'standalone'
           trace "building standalone worker"
 

data/lib/skylight/worker/collector.rb

@@ -7,7 +7,7 @@ module Skylight
 
       ENDPOINT     = '/report'.freeze
       CONTENT_TYPE = 'content-type'.freeze
-      SKYLIGHT_V1  = 'application/x-skylight-report-v1'.freeze
+      SKYLIGHT_V2  = 'application/x-skylight-report-v2'.freeze
 
       include Util::Logging
 
@@ -20,7 +20,6 @@ module Skylight
         @size        = config[:'agent.sample']
         @batch       = nil
         @interval    = config[:'agent.interval']
-        @buf         = ""
         @refresh_at  = 0
         @http_auth   = Util::HTTP.new(config, :accounts)
         @http_report = nil
@@ -29,7 +28,7 @@ module Skylight
         t { fmt "starting collector; interval=%d; size=%d", @interval, @size }
       end
 
-      def handle(msg, now = Util::Clock.secs)
+      def handle(msg, now = Util::Clock.absolute_secs)
         @batch ||= new_batch(now)
 
         if should_refresh_token?(now)
@@ -50,10 +49,10 @@ module Skylight
         return true unless msg
 
         case msg
-        when Messages::Trace
+        when Messages::TraceEnvelope
           t { fmt "collector received trace" }
           @batch.push(msg)
-        when Messages::Error
+        when Error
           send_error(msg)
         else
           debug "Received unknown message; class=%s", msg.class.to_s
@@ -96,7 +95,7 @@ module Skylight
       def finish
         t { fmt "collector finishing up" }
 
-        now = Util::Clock.secs
+        now = Util::Clock.absolute_secs
 
         if should_refresh_token?(now)
           refresh_report_token(now)
@@ -114,8 +113,7 @@ module Skylight
 
         debug "flushing batch; size=%d", batch.sample.count
 
-        @buf.clear
-        @http_report.post(ENDPOINT, batch.encode(@buf), CONTENT_TYPE => SKYLIGHT_V1)
+        @http_report.post(ENDPOINT, batch.encode, CONTENT_TYPE => SKYLIGHT_V2)
       end
 
       def refresh_report_token(now)
@@ -137,13 +135,15 @@ module Skylight
         tok = res.body['session']
         tok = tok['token'] if tok
 
+        t { "New token: #{tok}" }
+
         if tok
-          @refresh_at  = now + 30
+          @refresh_at  = now + 1800 # 30 minutes
           @http_report = Util::HTTP.new(config, :report)
           @http_report.authentication = tok
         else
           if @http_report
-            @refresh_at = now + 30
+            @refresh_at = now + 60
           end
           warn "server did not return a session token"
         end
@@ -157,6 +157,7 @@ module Skylight
       end
 
       def has_report_token?(now)
+        return true if config.ignore_token?
         return unless @http_report
         now < @refresh_at + (3600 * 3 - 660)
       end
@@ -184,6 +185,7 @@ module Skylight
 
         def should_flush?(now)
           return true if @config.constant_flush?
+          t { "Checking if should flush at #{now} (@flush_at is #{@flush_at})" }
           now >= @flush_at
         end
 
@@ -193,37 +195,23 @@ module Skylight
 
         def push(trace)
           # Count it
-          @counts[trace.endpoint] += 1
+          @counts[trace.endpoint_name] += 1
           # Push the trace into the sample
           @sample << trace
         end
 
-        def encode(buf)
-          endpoints = {}
+        def encode
+          batch = Skylight::Batch.native_new(from, config[:hostname])
 
           sample.each do |trace|
-            unless name = trace.endpoint
-              debug "trace missing name -- dropping"
-              next
-            end
-
-            trace.endpoint = nil
-
-            ep = (endpoints[name] ||= Messages::Endpoint.new(
-              name: name, traces: [], count: @counts[name]))
-
-            ep.traces << trace
+            batch.native_move_in(trace.data)
           end
 
-          t { fmt "encoding batch; endpoints=%p", endpoints.keys }
-
-          Messages::Batch.new(
-            timestamp: from,
-            hostname:  config[:hostname],
-            endpoints: endpoints.values).
-            encode(buf)
+          @counts.each do |endpoint_name,count|
+            batch.native_set_endpoint_count(endpoint_name, count)
+          end
 
-          buf
+          batch.native_serialize
         end
       end
 

data/lib/skylight/worker/connection.rb

@@ -44,7 +44,7 @@ module Skylight
       def maybe_read_message
         if @len && @buf.bytesize >= @len + FRAME_HDR_LEN
           mid   = read_message_id
-          klass = Messages.get(mid)
+          klass = Messages::ID_TO_KLASS.fetch(mid)
           data  = @buf[FRAME_HDR_LEN, @len]
           @buf  = @buf[(FRAME_HDR_LEN + @len)..-1] || ""
 
@@ -59,7 +59,7 @@ module Skylight
           end
 
           begin
-            return klass.decode(data)
+            return klass.deserialize(data)
           rescue Exception => e
             # reraise protobuf decoding exceptions
             raise IpcProtoError, e.message

data/lib/skylight/worker/embedded.rb

@@ -1,6 +1,24 @@
 module Skylight
   module Worker
     class Embedded
+      def initialize(collector)
+        @collector = collector
+      end
+
+      def spawn
+        @collector.spawn
+      end
+
+      def shutdown
+        @collector.shutdown
+      end
+
+      def submit(msg)
+        decoder = Messages::ID_TO_KLASS.fetch(Messages::KLASS_TO_ID.fetch(msg.class))
+        msg = decoder.deserialize(msg.serialize)
+
+        @collector.submit(msg)
+      end
     end
   end
 end

data/lib/skylight/worker/server.rb

@@ -221,12 +221,12 @@ module Skylight
         case msg
         when nil
           return
-        when Messages::Hello
+        when Hello
           if msg.newer?
             info "newer version of agent deployed - restarting; curr=%s; new=%s", VERSION, msg.version
             reload(msg)
           end
-        when Messages::Base
+        when Messages::TraceEnvelope, Error
           t { "received message" }
           @collector.submit(msg)
         when :unknown
@@ -327,7 +327,7 @@ module Skylight
 
       def get_memory_usage
         `ps -o rss= -p #{Process.pid}`.to_i / 1024
-      rescue Errno::ENOENT, Errno::EINTR
+      rescue Errno::ENOENT
         0
       end
     end

data/lib/skylight/worker/standalone.rb

@@ -63,7 +63,7 @@ module Skylight
       end
 
       def submit(msg)
-        unless msg.respond_to?(:encode)
+        unless msg.respond_to?(:encode) || msg.respond_to?(:native_serialize)
           raise ArgumentError, "message not encodable"
         end
 
@@ -217,11 +217,13 @@ module Skylight
       end
 
       def write_msg(sock, msg)
-        buf   = msg.encode.to_s
-        frame = [ msg.message_id, buf.bytesize ].pack("LL")
+        t { "writing a #{msg.class} on the wire" }
+        id = Messages::KLASS_TO_ID.fetch(msg.class)
+        buf = msg.serialize
 
-        write(sock, frame) &&
-          write(sock, buf)
+        frame = [ id, buf.bytesize ].pack("LL")
+
+        write(sock, frame) && write(sock, buf)
       end
 
       SOCK_TIMEOUT_VAL = [ 0, 0.01 * 1_000_000 ].pack("l_2")
@@ -354,11 +356,8 @@ module Skylight
         end
       end
 
-      def build_hello(config_version = nil)
-        Messages::Hello.new(
-          version: VERSION,
-          cmd:     SUBPROCESS_CMD,
-          config:  config_version)
+      def build_hello
+        Messages::Hello.build(VERSION, SUBPROCESS_CMD)
       end
 
       def build_queue

data/lib/sql_lexer.rb

@@ -2,5 +2,4 @@ require "sql_lexer/version"
 require "sql_lexer/lexer"
 
 module SqlLexer
-  # Your code goes here...
 end

data/lib/sql_lexer/lexer.rb

@@ -29,6 +29,8 @@ module SqlLexer
     StartString   = %Q<'>
     StartDigit    = %q<[\p{Digit}\.]>
 
+    StartSelect   = %Q<SELECT(?=(?:[#{WS}]|#{OpPart}))>
+
     # Binds that are also IDs do not need to be included here, since AfterOp (which uses StartBind)
     # also checks for StartAnyId
     StartBind     = %Q<#{StartString}|#{StartDigit}|#{SpecialOps}>
@@ -85,7 +87,9 @@ module SqlLexer
     TkArray       = %r<#{ArrayOp}>iu
     TkArrayIndex  = %r<#{ArrayIndexOp}>iu
     TkSpecialOp   = %r<#{SpecialOps}>iu
-    TkStartSelect = %r<SELECT(?=(?:[#{WS}]|#{OpPart}))>iu
+    TkStartSelect = %r<#{StartSelect}>iu
+    TkStartSubquery = %r<\(#{OptWS}#{StartSelect}>iu
+    TkCloseParen  = %r<#{OptWS}\)>u
 
     STATE_HANDLERS = {
       begin:       :process_begin,
@@ -97,6 +101,7 @@ module SqlLexer
       table_name:  :process_table_name,
       end:         :process_end,
       special:     :process_special,
+      subquery:    :process_subquery,
       in:          :process_in,
       array:       :process_array
     }
@@ -351,8 +356,12 @@ module SqlLexer
     def process_special
       if @scanner.skip(TkIn)
         @scanner.skip(TkOptWS)
-        @scanner.skip(/\(/u)
-        @state = :in
+        if @scanner.skip(TkStartSubquery)
+          @state = :subquery
+        else
+          @scanner.skip(/\(/u)
+          @state = :in
+        end
       elsif @scanner.skip(TkArray)
         @scanner.skip(/\[/u)
         @state = :array
@@ -362,11 +371,48 @@ module SqlLexer
         else
           @state = :end
         end
+      elsif @scanner.skip(TkStartSubquery)
+        @state = :subquery
       elsif @scanner.skip(TkArrayIndex)
         @state = :tokens
       end
     end
 
+    def process_subquery
+      nest = 1
+      iterations = 0
+
+      while nest > 0
+        iterations += 1
+
+        if iterations > 10_000
+          raise "The SQL '#{@scanner.string}' could not be parsed because of too many iterations in subquery"
+        end
+
+        if @debug
+          p @state
+          p @scanner
+          p nest
+          p @scanner.peek(1)
+        end
+
+        if @scanner.skip(TkStartSubquery)
+          nest += 1
+          @state = :tokens
+        elsif @scanner.skip(TkCloseParen)
+          nest -= 1
+          break if nest.zero?
+          @state = :tokens
+        elsif @state == :subquery
+          @state = :tokens
+        end
+
+        __send__ STATE_HANDLERS[@state]
+      end
+
+      @state = :tokens
+    end
+
     def process_in
       nest = 1
       iterations = 0
@@ -390,7 +436,7 @@ module SqlLexer
         if @scanner.skip(/\(/u)
           nest += 1
           process_tokens
-        elsif @scanner.skip(/\)/u)
+        elsif @scanner.skip(TkCloseParen)
           nest -= 1
           break if nest.zero?
           process_tokens