skylight 0.2.7 → 0.3.0.rc.3

data/lib/skylight.rb

@@ -2,6 +2,19 @@ require 'rbconfig'
 require 'socket'
 require 'skylight/version'
 
+begin
+  unless ENV["SKYLIGHT_DISABLE_AGENT"]
+    require 'skylight_native'
+    require 'skylight/native'
+    has_native_ext = true
+  end
+rescue LoadError
+  puts "[SKYLIGHT] The Skylight native extension wasn't found. Skylight is not running."
+  raise if ENV.key?("SKYLIGHT_REQUIRED")
+end
+
+if has_native_ext
+
 module Skylight
   TRACE_ENV_KEY      = 'SKYLIGHT_ENABLE_TRACE_LOGS'.freeze
   STANDALONE_ENV_KEY = 'SKYLIGHT_STANDALONE'.freeze
@@ -32,7 +45,8 @@ module Skylight
   autoload :Worker,       'skylight/worker'
 
   module Util
-    autoload :Clock,         'skylight/util/clock'
+    require 'skylight/util/clock'
+
     autoload :Gzip,          'skylight/util/gzip'
     autoload :HTTP,          'skylight/util/http'
     autoload :Inflector,     'skylight/util/inflector'
@@ -46,9 +60,6 @@ module Skylight
     autoload :HTTP, 'skylight/formatters/http'
   end
 
-  # ==== Vendor ====
-  autoload :Beefcake, 'skylight/vendor/beefcake'
-
   # ==== Exceptions ====
   class IpcProtoError    < RuntimeError; end
   class WorkerStateError < RuntimeError; end
@@ -77,19 +88,24 @@ module Skylight
     Instrumenter.stop!(*args)
   end
 
-  def self.trace(title=nil, desc=nil, annot=nil)
+  def self.trace(endpoint=nil, cat=nil, title=nil)
     unless inst = Instrumenter.instance
       return yield if block_given?
       return
     end
 
     if block_given?
-      inst.trace(title, desc, annot) { yield }
+      inst.trace(endpoint, cat, title) { yield }
     else
-      inst.trace(title, desc, annot)
+      inst.trace(endpoint, cat, title)
     end
   end
 
+  def self.done(span)
+    return unless inst = Instrumenter.instance
+    inst.done(span)
+  end
+
   def self.instrument(opts = DEFAULT_OPTIONS)
     unless inst = Instrumenter.instance
       return yield if block_given?
@@ -135,3 +151,30 @@ module Skylight
 
   require 'skylight/probes'
 end
+
+else
+
+module Skylight
+  def self.start!(*)
+  end
+
+  def self.stop!(*)
+  end
+
+  def self.trace(*)
+    yield if block_given?
+  end
+
+  def self.done(*)
+  end
+
+  def self.instrument(*)
+    yield if block_given?
+  end
+
+  def self.disable(*)
+    yield if block_given?
+  end
+end
+
+end

data/lib/skylight/config.rb

@@ -40,7 +40,9 @@ module Skylight
       'ACCOUNTS_SSL'        => :'accounts.ssl',
       'ACCOUNTS_DEFLATE'    => :'accounts.deflate',
       'ME_AUTHENTICATION'   => :'me.authentication',
-      'ME_CREDENTIALS_PATH' => :'me.credentials_path' }
+      'ME_CREDENTIALS_PATH' => :'me.credentials_path',
+      'TEST_CONSTANT_FLUSH' => :'test.constant_flush',
+      'TEST_IGNORE_TOKEN'   => :'test.ignore_token' }
 
     # Default values for Skylight configuration keys
     DEFAULTS = {
@@ -261,6 +263,10 @@ authentication: #{self[:authentication]}
       get('test.constant_flush')
     end
 
+    def ignore_token?
+      get('test.ignore_token')
+    end
+
     def root
       self[:root] || Dir.pwd
     end

data/lib/skylight/helpers.rb

@@ -39,7 +39,7 @@ module Skylight
         title    = (opts[:title] || title).to_s
         desc     = opts[:description].to_s if opts[:description]
 
-        klass.class_eval <<-RUBY
+        klass.class_eval <<-RUBY, __FILE__, __LINE__ + 1
           alias_method :"#{name}_before_instrument", :"#{name}"
 
           def #{name}(*args, &blk)
@@ -51,7 +51,7 @@ module Skylight
             begin
               #{name}_before_instrument(*args, &blk)
             ensure
-              span.done if span
+              Skylight.done(span) if span
             end
           end
         RUBY

data/lib/skylight/instrumenter.rb

@@ -97,7 +97,7 @@ module Skylight
       end
 
       begin
-        trace = Messages::Trace::Builder.new(self, endpoint, Util::Clock.micros, cat, title, desc, annot)
+        trace = Messages::Trace::Builder.new(self, endpoint, Util::Clock.nanos, cat, title, desc, annot)
       rescue Exception => e
         log_error e.message
         t { e.backtrace.join("\n") }
@@ -137,6 +137,11 @@ module Skylight
       self.class.match?(string, regex)
     end
 
+    def done(span)
+      return unless trace = @trace_info.current
+      trace.done(span)
+    end
+
     def instrument(cat, title=nil, desc=nil, annot=nil)
       unless trace = @trace_info.current
         return yield if block_given?
@@ -163,7 +168,7 @@ module Skylight
       begin
         yield sp
       ensure
-        sp.done
+        trace.done(sp)
       end
     end
 
@@ -186,7 +191,7 @@ module Skylight
     def error(type, description, details=nil)
       t { fmt "processing error; type=%s; description=%s", type, description }
 
-      message = Skylight::Messages::Error.new(type: type, description: description, details: details && details.to_json)
+      message = Skylight::Messages::Error.build(type, description, details && details.to_json)
 
       unless @worker.submit(message)
         warn "failed to submit error to worker"
@@ -194,8 +199,7 @@ module Skylight
     end
 
     def process(trace)
-      t { fmt "processing trace; spans=%d; duration=%d",
-            trace.spans.length, trace.spans[-1].duration }
+      t { fmt "processing trace" }
       unless @worker.submit(trace)
         warn "failed to submit trace to worker"
       end

data/lib/skylight/messages.rb

@@ -1,20 +1,20 @@
 module Skylight
   module Messages
-    def self.get(id)
-      (@id_map ||= {})[id]
-    end
-
-    def self.set(id, klass)
-      (@id_map ||= {})[id] = klass
-    end
-
-    require 'skylight/messages/annotation'
-    require 'skylight/messages/event'
-    require 'skylight/messages/span'
     require 'skylight/messages/trace'
-    require 'skylight/messages/endpoint'
-    require 'skylight/messages/batch'
     require 'skylight/messages/hello'
     require 'skylight/messages/error'
+    require 'skylight/messages/trace_envelope'
+
+    KLASS_TO_ID = {
+      Skylight::Hello => 1,
+      Skylight::Trace => 2,
+      Skylight::Error => 3
+    }
+
+    ID_TO_KLASS = {
+      1 => Skylight::Hello,
+      2 => Skylight::Messages::TraceEnvelope,
+      3 => Skylight::Error
+    }
   end
 end

data/lib/skylight/messages/error.rb

@@ -1,11 +1,15 @@
-require 'skylight/messages/base'
-
 module Skylight
   module Messages
-    class Error < Base
-      required :type,        :string, 1
-      required :description, :string, 2
-      optional :details,     :string, 3
+    class Error
+      def self.deserialize(buf)
+        decode(buf)
+      end
+
+      def self.build(group, description, details = nil)
+        Skylight::Error.native_new(group, description).tap do |error|
+          error.native_set_details(details) if details
+        end
+      end
     end
   end
 end

data/lib/skylight/messages/hello.rb

@@ -1,51 +1,10 @@
-require 'skylight/messages/base'
-
 module Skylight
   module Messages
-    class Hello < Base
-
-      DIGITS = /^\s*\d+\s*$/
-
-      required :version, :string, 1
-      optional :config,  :uint32, 2
-      repeated :cmd,     :string, 3
-
-      def newer?(other = VERSION)
-        other = split(other)
-        curr  = split(version)
-
-        [other.length, curr.length].max.times do |i|
-          next if other[i] == curr[i]
-          return true unless other[i]
-
-          if other[i] =~ DIGITS
-            if curr[i] =~ DIGITS
-              other_i = other[i].to_i
-              curr_i = curr[i].to_i
-
-              next if other_i == curr_i
-
-              return curr_i > other_i
-            else
-              return false
-            end
-          else
-            if curr[i] =~ DIGITS
-              return true
-            else
-              next if curr[i] == other[i]
-              return curr[i] > other[i]
-            end
-          end
+    class Hello
+      def self.build(version, cmd=[])
+        Skylight::Hello.native_new(version, 0).tap do |hello|
+          cmd.each { |part| hello.native_add_cmd_part(part) }
         end
-
-        false
-      end
-
-    private
-
-      def split(v)
-        v.split('.')
       end
 
     end

data/lib/skylight/messages/trace.rb

@@ -1,13 +1,6 @@
-require 'skylight/messages/base'
-
 module Skylight
   module Messages
-    class Trace < Base
-
-      required :uuid,     :string, 1
-      optional :endpoint, :string, 2
-      repeated :spans,    Span,    3
-
+    class Trace
       class Builder
         GC_CAT = 'noise.gc'.freeze
 
@@ -17,23 +10,23 @@ module Skylight
 
         def endpoint=(value)
           @endpoint = value.freeze
+          @native_builder.native_set_name(value)
         end
 
         def initialize(instrumenter, endpoint, start, cat, title=nil, desc=nil, annot=nil)
           raise ArgumentError, 'instrumenter is required' unless instrumenter
 
-          @instrumenter = instrumenter
-          @endpoint     = endpoint.freeze
-          @start        = start
-          @spans        = []
-          @stack        = []
-          @submitted    = false
+          start = normalize_time(start)
 
-          # Tracks the AS::N stack
-          @notifications = []
+          @native_builder = ::Skylight::Trace.native_new(start, "TODO")
+          @native_builder.native_set_name(endpoint)
 
-          # Track time
-          @last_seen_time = start
+          @instrumenter  = instrumenter
+          @endpoint      = endpoint.freeze
+          @submitted     = false
+          @start         = start
+
+          @notifications = []
 
           if Hash === title
             annot = title
@@ -43,26 +36,41 @@ module Skylight
             desc = nil
           end
 
-          # Create the root node
-          @root = start(@start, cat, title, desc, annot)
-          @gc   = config.gc.track
+          # create the root node
+          @root = @native_builder.native_start_span(@start, cat)
+          @native_builder.native_span_set_title(@root, title) if title
+          @native_builder.native_span_set_description(@root, desc) if desc
+
+          @gc   = config.gc.track unless ENV.key?("SKYLIGHT_DISABLE_GC_TRACKING")
+        end
+
+        def serialize
+          raise "Can only serialize once" if @serialized
+          @serialized = true
+          @native_builder.native_serialize
         end
 
         def config
           @instrumenter.config
         end
 
-        def record(cat, *args)
-          annot = args.pop if Hash === args
-          title = args.shift
-          desc  = args.shift
-          now   = adjust_for_skew(Util::Clock.micros)
+        def record(cat, title=nil, desc=nil, annot=nil)
+          if Hash === title
+            annot = title
+            title = desc = nil
+          elsif Hash === desc
+            annot = desc
+            desc = nil
+          end
+
+          title.freeze
+          desc.freeze
 
           desc = @instrumenter.limited_description(desc)
 
-          sp = span(now - gc_time, cat, title, desc, annot)
-          inc_children
-          @spans << sp.build(0)
+          time = Util::Clock.nanos - gc_time
+
+          stop(start(time, cat, title, desc), time)
 
           nil
         end
@@ -80,7 +88,7 @@ module Skylight
           desc.freeze
 
           original_desc = desc
-          now           = adjust_for_skew(Util::Clock.micros)
+          now           = Util::Clock.nanos
           desc          = @instrumenter.limited_description(desc)
 
           if desc == Instrumenter::TOO_MANY_UNIQUES
@@ -94,7 +102,7 @@ module Skylight
 
         def done(span)
           return unless span
-          stop(span, adjust_for_skew(Util::Clock.micros) - gc_time)
+          stop(span, Util::Clock.nanos - gc_time)
         end
 
         def release
@@ -102,37 +110,27 @@ module Skylight
           @instrumenter.current_trace = nil
         end
 
-        def submit
-          return if @submitted
-
-          release
-          @submitted = true
-
-          now = adjust_for_skew(Util::Clock.micros)
-
-          # Pop everything that is left
-          while sp = pop
-            @spans << sp.build(relativize(now) - sp.started_at)
-          end
-
+        def traced
           time = gc_time
+          now = Util::Clock.nanos
 
           if time > 0
             t { fmt "tracking GC time; duration=%d", time }
-            noise = start(now - time, GC_CAT, nil, nil, {})
-            stop(noise, now)
+            stop(start(now - time, GC_CAT, nil, nil, {}), now)
           end
 
-          if sp = @stack.pop
-            @spans << sp.build(relativize(now) - sp.started_at)
-          end
+          stop(@root, now)
+        end
+
+        def submit
+          return if @submitted
+
+          release
+          @submitted = true
 
-          t = Trace.new(
-            uuid:     'TODO',
-            endpoint: endpoint,
-            spans:    spans)
+          traced
 
-          @instrumenter.process(t)
+          @instrumenter.process(@native_builder)
         rescue Exception => e
           error e
           t { e.backtrace.join("\n") }
@@ -140,63 +138,24 @@ module Skylight
 
       private
 
-        def start(time, cat, title, desc, annot)
-          sp = span(time, cat, title, desc, annot)
-
-          push(sp)
-
-          sp
+        def start(time, cat, title, desc, annot=nil)
+          span(normalize_time(time), cat, title, desc, annot)
         end
 
         def stop(span, time)
-          until span == (sp = pop)
-            return unless sp
-            @spans << sp.build(relativize(time) - sp.started_at)
-          end
-
-          @spans << span.build(relativize(time) - sp.started_at)
-
+          @native_builder.native_stop_span(span, normalize_time(time))
           nil
         end
 
-        def span(time, cat, title, desc, annot)
-          Span::Builder.new(
-            self, time, relativize(time),
-            cat, title, desc, annot)
-        end
-
-        def pop
-          return unless @stack.length > 1
-          @stack.pop
-        end
-
-        def push(sp)
-          inc_children
-          @stack << sp
-        end
-
-        def inc_children
-          return unless sp = @stack[-1]
-          sp.children += 1
+        def normalize_time(time)
+          time.to_i / 100_000
         end
 
-        def relativize(time)
-          if parent = @stack[-1]
-            ((time - parent.time) / 100).to_i
-          else
-            ((time - @start) / 100).to_i
-          end
-        end
-
-        # Sadely, we don't have access to a pure monotonic clock in ruby, so we
-        # need to cheat a little.
-        def adjust_for_skew(time)
-          if time <= @last_seen_time
-            return @last_seen_time
-          end
-
-          @last_seen_time = time
-          time
+        def span(time, cat, title=nil, desc=nil, annot=nil)
+          sp = @native_builder.native_start_span(time, cat.to_s)
+          @native_builder.native_span_set_title(sp, title.to_s) if title
+          @native_builder.native_span_set_description(sp, desc.to_s) if desc
+          sp
         end
 
         def gc_time