skylight 0.0.2

data/lib/skylight/util/atomic.rb

@@ -0,0 +1,73 @@
+module Skylight
+  module Util
+    class AtomicRef
+      def initialize(v = nil)
+        @v = v
+        @m = Mutex.new
+      end
+
+      def get
+        @m.synchronize { @v }
+      end
+
+      def set(v)
+        @m.synchronize { @v = v }
+      end
+
+      def compare_and_set(expect, v)
+        @m.synchronize do
+          return false unless @v == expect
+          @v = v
+        end
+
+        true
+      end
+
+      def get_and_set(v)
+        while true
+          c = get
+          return c if compare_and_set(c, v)
+        end
+      end
+    end
+
+    class AtomicInteger < AtomicRef
+
+      def initialize(v = 0)
+        super(v)
+      end
+
+      def add_and_get(delta)
+        while true
+          c = get
+          n = c + delta
+          return n if compare_and_set(c, n)
+        end
+      end
+
+      def increment_and_get
+        add_and_get(1)
+      end
+
+      def decrement_and_get
+        add_and_get(-1)
+      end
+
+      def get_and_add(delta)
+        while true
+          c = get
+          n = c + delta
+          return c if compare_and_set(c, n)
+        end
+      end
+
+      def get_and_increment
+        get_and_add(1)
+      end
+
+      def get_and_decrement
+        get_and_add(-1)
+      end
+    end
+  end
+end

data/lib/skylight/util/bytes.rb

@@ -0,0 +1,40 @@
+module Skylight
+  module Util
+    class OutOfRangeError < RuntimeError; end
+
+    module Bytes
+      # Helper consts
+      MinUint64 =  0
+      MaxUint64 =  (1<<64)-1
+      MinInt64  = -(1<<63)
+      MaxInt64  =  (1<<63)-1
+
+      # varints
+      def append_uint64(buf, n)
+        if n < MinUint64 || n > MaxUint64
+          raise OutOfRangeError, n
+        end
+
+        while true
+          bits = n & 0x7F
+          n >>= 7
+
+          if n == 0
+            return buf << bits
+          end
+
+          buf << (bits | 0x80)
+        end
+      end
+
+      def str_bytesize(str)
+        str.bytesize
+      end
+
+      def append_string(buf, str)
+        append_uint64(buf, str_bytesize(str))
+        buf << str
+      end
+    end
+  end
+end

data/lib/skylight/util/clock.rb

@@ -0,0 +1,37 @@
+module Skylight
+  module Util
+    class Clock
+      MICROSEC_PER_SEC = 1.to_f / 1_000_000
+
+      # Resolution is in seconds
+      def initialize(resolution)
+        @resolution = resolution
+        @usec_mult  = MICROSEC_PER_SEC / resolution
+      end
+
+      def now
+        at(Time.now)
+      end
+
+      def at(time)
+        sec  = time.to_i / @resolution
+        usec = time.usec * @usec_mult
+        (sec + usec).floor
+      end
+
+      def convert(secs)
+        (secs / @resolution).floor
+      end
+
+      def to_seconds(clock_val)
+        (clock_val * @resolution).floor
+      end
+    end
+
+    @clock = Clock.new(0.0001)
+
+    def self.clock
+      @clock
+    end
+  end
+end

data/lib/skylight/util/ewma.rb

@@ -0,0 +1,32 @@
+module Skylight
+  module Util
+    class EWMA
+
+      attr_reader :rate
+
+      def initialize(alpha)
+        @alpha     = alpha
+        @uncounted = AtomicInteger.new
+        @rate      = nil
+      end
+
+      def update(n)
+        @uncounted.add_and_get(n)
+      end
+
+      # Mark the passage of time and decay the current rate accordingly.
+      # This method is obviously not thread-safe as is expected to be
+      # invoked once every interval
+      def tick()
+        count = @uncounted.get_and_set(0)
+        instantRate = count
+
+        if rate
+          rate += (alpha * (instantRate - rate))
+        else
+          rate = instantRate
+        end
+      end
+    end
+  end
+end

data/lib/skylight/util/gzip.rb

@@ -0,0 +1,15 @@
+require 'zlib'
+
+module Skylight
+  module Util
+    module Gzip
+      def self.compress(str)
+        output = StringIO.new
+        gz = Zlib::GzipWriter.new(output)
+        gz.write(str)
+        gz.close
+        output.string
+      end
+    end
+  end
+end

data/lib/skylight/util/queue.rb

@@ -0,0 +1,93 @@
+require 'thread'
+
+module Skylight
+  module Util
+    # Simple thread-safe queue backed by a ring buffer. Will only block when
+    # poping.
+    class Queue
+
+      def initialize(max)
+        unless max > 0
+          raise ArgumentError, "queue size must be positive"
+        end
+
+        @max     = max
+        @values  = [nil] * max
+        @consume = 0
+        @produce = 0
+        @waiting = []
+        @mutex   = Mutex.new
+      end
+
+      def empty?
+        @mutex.synchronize { __empty? }
+      end
+
+      def length
+        @mutex.synchronize { __length }
+      end
+
+      # Returns true if the item was queued, false otherwise
+      def push(obj)
+        @mutex.synchronize do
+          return false if __length == @max
+          @values[@produce] = obj
+          @produce = (@produce + 1) % @max
+
+          # Wakeup a blocked thread
+          begin
+            t = @waiting.shift
+            t.wakeup if t
+          rescue ThreadError
+            retry
+          end
+        end
+
+        true
+      end
+
+      def pop(timeout = nil)
+        if timeout && timeout < 0
+          raise ArgumentError, "timeout must be nil or >= than 0"
+        end
+
+        @mutex.synchronize do
+          if __empty?
+            if !timeout || timeout > 0
+              t = Thread.current
+              @waiting << t
+              @mutex.sleep(timeout)
+              # Ensure that the thread is not in the waiting list
+              @waiting.delete(t)
+            else
+              return
+            end
+          end
+
+          __pop unless __empty?
+        end
+      end
+
+    private
+
+      def __length
+        ((@produce - @consume) % @max)
+      end
+
+      def __empty?
+        @produce == @consume
+      end
+
+      def __pop
+        i = @consume
+        v = @values[i]
+
+        @values[i] = nil
+        @consume = (i + 1) % @max
+
+        return v
+      end
+
+    end
+  end
+end

data/lib/skylight/util/uniform_sample.rb

@@ -0,0 +1,63 @@
+module Skylight
+  module Util
+    class UniformSample
+      include Enumerable
+
+      attr_reader :size, :count
+
+      def initialize(size)
+        @size   = size
+        @count  = 0
+        @values = []
+      end
+
+      def clear
+        @count = 0
+        @values.clear
+        self
+      end
+
+      def length
+        @size < @count ? @size : @count
+      end
+
+      def empty?
+        @count == 0
+      end
+
+      def each
+        i  = 0
+        to = length
+
+        while i < to
+          yield @values[i]
+          i += 1
+        end
+
+        self
+      end
+
+      def <<(v)
+        if idx = increment!
+          @values[idx] = v
+        end
+
+        self
+      end
+
+    private
+
+      def increment!
+        c = (@count += 1)
+
+        if (c <= @size)
+          c - 1
+        else
+          r = rand(@count)
+          r if r < @size
+        end
+      end
+
+    end
+  end
+end

data/lib/skylight/util/uuid.rb

@@ -0,0 +1,33 @@
+module Skylight
+  module Util
+    class UUID
+      BYTE_SIZE   = 16
+      PREFIX_SIZE = 8
+
+      def self.gen(prefix = nil)
+        if prefix == nil
+          return SecureRandom.random_bytes(BYTE_SIZE)
+        end
+
+        if prefix.bytesize > PREFIX_SIZE
+          raise "UUID prefix must be less than 8 bytes"
+        end
+
+        # Does not fully conform with the spec
+        rnd = SecureRandom.random_bytes(BYTE_SIZE - prefix.bytesize)
+        new "#{prefix}#{rnd}"
+      end
+
+      attr_reader :bytes
+
+      def initialize(bytes)
+        @bytes = bytes
+      end
+
+      def to_s
+        @to_s ||= ""
+      end
+
+    end
+  end
+end

data/lib/skylight/version.rb

@@ -0,0 +1,3 @@
+module Skylight
+  VERSION = '0.0.2'
+end

data/lib/skylight/worker.rb

@@ -0,0 +1,232 @@
+module Skylight
+  class Worker
+    CONTENT_ENCODING = 'content-encoding'.freeze
+    CONTENT_LENGTH   = 'content-length'.freeze
+    CONTENT_TYPE     = 'content-type'.freeze
+    APPLICATION_JSON = 'application/json'.freeze
+    DIREWOLF_REPORT  = 'application/x-direwolf-report'.freeze
+    AUTHORIZATION    = 'authorization'.freeze
+    ENDPOINT         = '/report'.freeze
+    DEFLATE          = 'deflate'.freeze
+    GZIP             = 'gzip'.freeze
+    FLUSH_DELAY      = Util.clock.convert(0.5)
+
+    class Batch
+      attr_reader :from, :counts, :sample
+
+      def initialize(config, from, interval)
+        @config = config
+        @interval = interval
+        @from = from
+        @to = from + interval
+        @flush_at = @to + FLUSH_DELAY
+        @sample = Util::UniformSample.new(config.samples_per_interval)
+        @counts = Hash.new { |h,k| h[k] = 0 }
+      end
+
+      def should_flush?(now)
+        now >= @flush_at
+      end
+
+      def next_batch
+        Batch.new(@config, @to, @interval)
+      end
+
+      def empty?
+        @sample.empty?
+      end
+
+      def wants?(trace)
+        return trace.to >= @from && trace.to < @to
+      end
+
+      def push(trace)
+        # Count it
+        @counts[trace.endpoint] += 1
+        # Push the trace into the sample
+        @sample << trace
+      end
+    end
+
+    attr_reader :instrumenter, :connection
+
+    def initialize(instrumenter)
+      @instrumenter = instrumenter
+      @interval     = config.interval
+      @protocol     = config.protocol
+
+      reset
+    end
+
+    def start!
+      shutdown! if @thread
+      @thread = Thread.new { work }
+      self
+    end
+
+    def shutdown!
+      # Don't do anything if the worker isn't running
+      return self unless @thread
+
+      thread  = @thread
+      @thread = nil
+
+      @queue.push(:SHUTDOWN)
+
+      unless thread.join(5)
+        begin
+          # FORCE KILL!!
+          thread.kill
+        rescue ThreadError
+        end
+      end
+
+      reset
+      self
+    end
+
+    def submit(trace)
+      return unless @thread
+      @queue.push(trace)
+    end
+
+    # A worker iteration
+    def iter(msg, now=Util.clock.now)
+      unless @current_batch
+        interval = Util.clock.convert(@interval)
+        from = (now / interval) * interval
+
+        # If we're still accepting traces from the previous batch
+        # Create the previous interval instead
+        if now < from + FLUSH_DELAY
+          from -= interval
+        end
+
+        @current_batch = Batch.new(config, from, interval)
+        @next_batch    = @current_batch.next_batch
+      end
+
+      if msg == :SHUTDOWN
+        flush(@current_batch) if @current_batch
+        flush(@next_batch)    if @next_batch
+        return false
+      end
+
+      while @current_batch && @current_batch.should_flush?(now)
+        flush(@current_batch)
+        @current_batch = @next_batch
+        @next_batch = @current_batch.next_batch
+      end
+
+      if Trace === msg
+        debug "Received trace"
+        if @current_batch.wants?(msg)
+          @current_batch.push(msg)
+        elsif @next_batch.wants?(msg)
+          @next_batch.push(msg)
+        else
+          # Seems bad bro
+        end
+      end
+
+      true
+    end
+
+  private
+
+    def config
+      @instrumenter.config
+    end
+
+    def logger
+      config.logger
+    end
+
+    def reset
+      @queue = Util::Queue.new(config.max_pending_traces)
+    end
+
+    def work
+      loop do
+        msg = @queue.pop(@interval.to_f / 20)
+        success = iter(msg)
+        return if !success
+      end
+    rescue Exception => e
+      logger.error "[SKYLIGHT] #{e.message} - #{e.class} - #{e.backtrace.first}"
+      if logger.debug?
+        logger.debug(e.backtrace.join("\n"))
+      end
+    end
+
+    attr_reader :sample_starts_at, :interval
+
+    def flush(batch)
+      return if batch.empty?
+      # Skip if there is no authentication token
+      return unless config.authentication_token
+
+      debug "Flushing: #{batch}"
+
+      body = ''
+      # write the body
+      @protocol.write(body, batch.from, batch.counts, batch.sample)
+
+      if config.deflate?
+        body = Util::Gzip.compress(body)
+      end
+
+      # send
+      http_post(body)
+    end
+
+    def http_post(body)
+      req = http_request(body.bytesize)
+      req.body = body
+
+      debug "Posting report to server"
+      http = Net::HTTP.new config.host, config.port
+      http.use_ssl = true if config.ssl?
+
+      http.start do |http|
+        resp = http.request req
+
+        unless resp.code == '200'
+          debug "Server responded with #{resp.code}"
+        end
+      end
+
+      true
+    rescue => e
+      logger.error "[SKYLIGHT] #{e.message} - #{e.class} - #{e.backtrace.first}"
+      if logger.debug?
+        logger.debug(e.backtrace.join("\n"))
+      end
+    end
+
+    def http_request(length)
+      hdrs = {}
+
+      hdrs[CONTENT_LENGTH] = length.to_s
+      hdrs[AUTHORIZATION]  = config.authentication_token
+      hdrs[CONTENT_TYPE]   = APPLICATION_JSON
+
+      if config.deflate?
+        hdrs[CONTENT_ENCODING] = GZIP
+      end
+
+      Net::HTTPGenericRequest.new \
+        'POST',   # Request method
+        true,     # There is a request body
+        true,     # There is a response body
+        ENDPOINT, # Endpoint
+        hdrs
+    end
+
+    def debug(msg)
+      return unless logger && logger.debug?
+      logger.debug "[SKYLIGHT] #{msg}"
+    end
+
+  end
+end