skylight 0.0.2

data/README.md

@@ -0,0 +1,20 @@
+# Direwolf Agent
+
+Collect instrumentation from a rails application and send it off to the
+direwolf servers.
+
+## Traces
+
+An conceptual overview of a trace.
+
+  * Trace ID (UUID)
+  * Tiers
+    * Name
+    * Spans (max 256 unique per tier)
+      * Category (multi-level, example: cache.redis)
+      * Description
+      * Annotations
+
+## TODO
+
+  * Cap the max number of spans per trace (2048k?)

data/lib/skylight.rb

@@ -0,0 +1,33 @@
+require 'thread'
+require 'socket'
+require 'openssl'
+require 'net/http'
+require 'active_support/notifications'
+
+module Skylight
+  class Error < RuntimeError; end
+
+  # First require all util files
+  require 'skylight/util/atomic'
+  require 'skylight/util/bytes'
+  require 'skylight/util/clock'
+  require 'skylight/util/ewma'
+  require 'skylight/util/gzip'
+  require 'skylight/util/queue'
+  require 'skylight/util/uniform_sample'
+  require 'skylight/util/uuid'
+
+  # Then require the rest
+  require 'skylight/compat'
+  require 'skylight/config'
+  require 'skylight/instrumenter'
+  require 'skylight/middleware'
+  require 'skylight/json_proto'
+  require 'skylight/subscriber'
+  require 'skylight/trace'
+  require 'skylight/worker'
+end
+
+if defined?(Rails)
+  require 'skylight/railtie'
+end

data/lib/skylight/compat.rb

@@ -0,0 +1,9 @@
+module Skylight
+  # Catch-up out of date software
+  #
+  # TODO: Add smarter detection
+  if true
+    require 'skylight/compat/notifications'
+  end
+
+end

data/lib/skylight/compat/notifications.rb

@@ -0,0 +1,175 @@
+require 'skylight/compat/notifications/instrumenter'
+require 'skylight/compat/notifications/fanout'
+
+module ActiveSupport
+  # = Notifications
+  #
+  # <tt>ActiveSupport::Notifications</tt> provides an instrumentation API for Ruby.
+  #
+  # == Instrumenters
+  #
+  # To instrument an event you just need to do:
+  #
+  #   ActiveSupport::Notifications.instrument("render", :extra => :information) do
+  #     render :text => "Foo"
+  #   end
+  #
+  # That executes the block first and notifies all subscribers once done.
+  #
+  # In the example above "render" is the name of the event, and the rest is called
+  # the _payload_. The payload is a mechanism that allows instrumenters to pass
+  # extra information to subscribers. Payloads consist of a hash whose contents
+  # are arbitrary and generally depend on the event.
+  #
+  # == Subscribers
+  #
+  # You can consume those events and the information they provide by registering
+  # a subscriber. For instance, let's store all "render" events in an array:
+  #
+  #   events = []
+  #
+  #   ActiveSupport::Notifications.subscribe("render") do |*args|
+  #     events << ActiveSupport::Notifications::Event.new(*args)
+  #   end
+  #
+  # That code returns right away, you are just subscribing to "render" events.
+  # The block is saved and will be called whenever someone instruments "render":
+  #
+  #   ActiveSupport::Notifications.instrument("render", :extra => :information) do
+  #     render :text => "Foo"
+  #   end
+  #
+  #   event = events.first
+  #   event.name      # => "render"
+  #   event.duration  # => 10 (in milliseconds)
+  #   event.payload   # => { :extra => :information }
+  #
+  # The block in the <tt>subscribe</tt> call gets the name of the event, start
+  # timestamp, end timestamp, a string with a unique identifier for that event
+  # (something like "535801666f04d0298cd6"), and a hash with the payload, in
+  # that order.
+  #
+  # If an exception happens during that particular instrumentation the payload will
+  # have a key <tt>:exception</tt> with an array of two elements as value: a string with
+  # the name of the exception class, and the exception message.
+  #
+  # As the previous example depicts, the class <tt>ActiveSupport::Notifications::Event</tt>
+  # is able to take the arguments as they come and provide an object-oriented
+  # interface to that data.
+  #
+  # It is also possible to pass an object as the second parameter passed to the
+  # <tt>subscribe</tt> method instead of a block:
+  #
+  #   module ActionController
+  #     class PageRequest
+  #       def call(name, started, finished, unique_id, payload)
+  #         Rails.logger.debug ["notification:", name, started, finished, unique_id, payload].join(" ")
+  #       end
+  #     end
+  #   end
+  #
+  #   ActiveSupport::Notifications.subscribe('process_action.action_controller', ActionController::PageRequest.new)
+  #
+  # resulting in the following output within the logs including a hash with the payload:
+  #
+  #   notification: process_action.action_controller 2012-04-13 01:08:35 +0300 2012-04-13 01:08:35 +0300 af358ed7fab884532ec7 {
+  #      :controller=>"Devise::SessionsController",
+  #      :action=>"new",
+  #      :params=>{"action"=>"new", "controller"=>"devise/sessions"},
+  #      :format=>:html,
+  #      :method=>"GET",
+  #      :path=>"/login/sign_in",
+  #      :status=>200,
+  #      :view_runtime=>279.3080806732178,
+  #      :db_runtime=>40.053
+  #    }
+  #
+  # You can also subscribe to all events whose name matches a certain regexp:
+  #
+  #   ActiveSupport::Notifications.subscribe(/render/) do |*args|
+  #     ...
+  #   end
+  #
+  # and even pass no argument to <tt>subscribe</tt>, in which case you are subscribing
+  # to all events.
+  #
+  # == Temporary Subscriptions
+  #
+  # Sometimes you do not want to subscribe to an event for the entire life of
+  # the application. There are two ways to unsubscribe.
+  #
+  # WARNING: The instrumentation framework is designed for long-running subscribers,
+  # use this feature sparingly because it wipes some internal caches and that has
+  # a negative impact on performance.
+  #
+  # === Subscribe While a Block Runs
+  #
+  # You can subscribe to some event temporarily while some block runs. For
+  # example, in
+  #
+  #   callback = lambda {|*args| ... }
+  #   ActiveSupport::Notifications.subscribed(callback, "sql.active_record") do
+  #     ...
+  #   end
+  #
+  # the callback will be called for all "sql.active_record" events instrumented
+  # during the execution of the block. The callback is unsubscribed automatically
+  # after that.
+  #
+  # === Manual Unsubscription
+  #
+  # The +subscribe+ method returns a subscriber object:
+  #
+  #   subscriber = ActiveSupport::Notifications.subscribe("render") do |*args|
+  #     ...
+  #   end
+  #
+  # To prevent that block from being called anymore, just unsubscribe passing
+  # that reference:
+  #
+  #   ActiveSupport::Notifications.unsubscribe(subscriber)
+  #
+  # == Default Queue
+  #
+  # Notifications ships with a queue implementation that consumes and publish events
+  # to log subscribers in a thread. You can use any queue implementation you want.
+  #
+  module Notifications
+    class << self
+      attr_accessor :notifier
+
+      def publish(name, *args)
+        notifier.publish(name, *args)
+      end
+
+      def instrument(name, payload = {}, &blk)
+        if notifier.listening?(name)
+          instrumenter.instrument(name, payload, &blk)
+        else
+          yield payload if block_given?
+        end
+      end
+
+      def subscribe(*args, &block)
+        notifier.subscribe(*args, &block)
+      end
+
+      def subscribed(callback, *args, &block)
+        subscriber = subscribe(*args, &callback)
+        yield
+      ensure
+        unsubscribe(subscriber)
+      end
+
+      def unsubscribe(args)
+        notifier.unsubscribe(args)
+      end
+
+      def instrumenter
+        Thread.current[:"instrumentation_#{notifier.object_id}"] ||= Instrumenter.new(notifier)
+      end
+    end
+
+    self.notifier = Fanout.new
+  end
+end

data/lib/skylight/compat/notifications/fanout.rb

@@ -0,0 +1,166 @@
+require 'mutex_m'
+
+module ActiveSupport
+  module Notifications
+    # This is a default queue implementation that ships with Notifications.
+    # It just pushes events to all registered log subscribers.
+    #
+    # This class is thread safe. All methods are reentrant.
+    class Fanout
+      include Mutex_m
+
+      def initialize
+        @subscribers = []
+        @listeners_for = {}
+        super
+      end
+
+      def subscribe(pattern = nil, block = Proc.new)
+        subscriber = Subscribers.new pattern, block
+        synchronize do
+          @subscribers << subscriber
+          @listeners_for.clear
+        end
+        subscriber
+      end
+
+      def unsubscribe(subscriber)
+        synchronize do
+          @subscribers.reject! { |s| s.matches?(subscriber) }
+          @listeners_for.clear
+        end
+      end
+
+      def start(name, id, payload)
+        listeners_for(name).each { |s| s.start(name, id, payload) }
+      end
+
+      def finish(name, id, payload)
+        listeners_for(name).each { |s| s.finish(name, id, payload) }
+      end
+
+      def measure(name, id, payload)
+        listeners_for(name).each { |s| s.measure(name, id, payload) }
+      end
+
+      def publish(name, *args)
+        listeners_for(name).each { |s| s.publish(name, *args) }
+      end
+
+      def listeners_for(name)
+        synchronize do
+          @listeners_for[name] ||= @subscribers.select { |s| s.subscribed_to?(name) }
+        end
+      end
+
+      def listening?(name)
+        listeners_for(name).any?
+      end
+
+      # This is a sync queue, so there is no waiting.
+      def wait
+      end
+
+      module Subscribers # :nodoc:
+        def self.new(pattern, listener)
+          if listener.respond_to?(:call)
+            subscriber = Timed.new pattern, listener
+          elsif listener.respond_to?(:measure)
+            subscriber = Evented.new pattern, listener
+          else
+            subscriber = LegacyEvented.new pattern, listener
+          end
+
+          unless pattern
+            AllMessages.new(subscriber)
+          else
+            subscriber
+          end
+        end
+
+        class Evented #:nodoc:
+          def initialize(pattern, delegate)
+            @pattern = pattern
+            @delegate = delegate
+          end
+
+          def start(name, id, payload)
+            @delegate.start name, id, payload
+          end
+
+          def finish(name, id, payload)
+            @delegate.finish name, id, payload
+          end
+
+          def measure(name, id, payload)
+            @delegate.measure(name, id, payload)
+          end
+
+          def subscribed_to?(name)
+            @pattern === name.to_s
+          end
+
+          def matches?(subscriber_or_name)
+            self === subscriber_or_name ||
+              @pattern && @pattern === subscriber_or_name
+          end
+        end
+
+        class LegacyEvented < Evented
+          def measure(name, id, payload)
+            start(name, id, payload)
+            finish(name, id, payload)
+          end
+        end
+
+        class Timed < LegacyEvented
+          def initialize(pattern, delegate)
+            @timestack = []
+            super
+          end
+
+          def publish(name, *args)
+            @delegate.call name, *args
+          end
+
+          def start(name, id, payload)
+            @timestack.push Time.now
+          end
+
+          def finish(name, id, payload)
+            started = @timestack.pop
+            @delegate.call(name, started, Time.now, id, payload)
+          end
+        end
+
+        class AllMessages # :nodoc:
+          def initialize(delegate)
+            @delegate = delegate
+          end
+
+          def start(name, id, payload)
+            @delegate.start name, id, payload
+          end
+
+          def finish(name, id, payload)
+            @delegate.finish name, id, payload
+          end
+
+          def measure(name, id, payload)
+            @delegate.measure name, id, payload
+          end
+
+          def publish(name, *args)
+            @delegate.publish name, *args
+          end
+
+          def subscribed_to?(name)
+            true
+          end
+
+          alias :matches? :===
+        end
+      end
+    end
+  end
+end

data/lib/skylight/compat/notifications/instrumenter.rb

@@ -0,0 +1,65 @@
+require 'securerandom'
+
+module ActiveSupport
+  module Notifications
+    # Instrumentors are stored in a thread local.
+    class Instrumenter
+      attr_reader :id
+
+      def initialize(notifier)
+        @id = unique_id
+        @notifier = notifier
+      end
+
+      # Instrument the given block by measuring the time taken to execute it
+      # and publish it. Notice that events get sent even if an error occurs
+      # in the passed-in block
+      def instrument(name, payload={})
+        if block_given?
+          @notifier.start(name, @id, payload)
+          begin
+            yield payload
+          rescue Exception => e
+            payload[:exception] = [e.class.name, e.message]
+            raise e
+          ensure
+            @notifier.finish(name, @id, payload)
+          end
+        else
+          @notifier.measure(name, @id, payload)
+        end
+      end
+
+      private
+        def unique_id
+          SecureRandom.hex(10)
+        end
+    end
+
+    class Event
+      attr_reader :name, :time, :transaction_id, :payload, :children
+      attr_accessor :end
+
+      def initialize(name, start, ending, transaction_id, payload)
+        @name           = name
+        @payload        = payload.dup
+        @time           = start
+        @transaction_id = transaction_id
+        @end            = ending
+        @children       = []
+      end
+
+      def duration
+        1000.0 * (self.end - time)
+      end
+
+      def <<(event)
+        @children << event
+      end
+
+      def parent_of?(event)
+        @children.include? event
+      end
+    end
+  end
+end