skylight-core 2.0.0.beta1

data/lib/skylight/core/probes/mongo.rb

@@ -0,0 +1,171 @@
+module Skylight::Core
+  module Probes
+    module Mongo
+      CAT = "db.mongo.command".freeze
+
+      class Probe
+        def install
+          ::Mongo::Monitoring::Global.subscribe(::Mongo::Monitoring::COMMAND, Subscriber.new)
+        end
+      end
+
+      class Subscriber
+        include Util::Logging
+
+        COMMANDS = [:insert, :find, :count, :distinct, :update, :findandmodify, :delete].freeze
+
+        COMMAND_NAMES = {
+          findandmodify: 'findAndModify'.freeze
+        }.freeze
+
+        def initialize
+          @events = {}
+        end
+
+        def started(event)
+          begin_instrumentation(event)
+        end
+
+        def succeeded(event)
+          end_instrumentation(event)
+        end
+
+        def failed(event)
+          end_instrumentation(event)
+        end
+
+        # For logging
+        def config
+          # Just log to the first instrumentable's config for now.
+          # FIXME: Revisit this
+          if (instrumentable = Fanout.registered.first)
+            if (instrumenter = instrumentable.instrumenter)
+              instrumenter.config
+            end
+          end
+        end
+
+        private
+
+          def begin_instrumentation(event)
+            return unless COMMANDS.include?(event.command_name.to_sym)
+
+            command_name = COMMAND_NAMES[event.command_name] || event.command_name
+
+            title = "#{event.database_name}.#{command_name}"
+
+            command = event.command
+
+            # Not sure if this will always exist
+            # Delete so the description will be less redundant
+            if target = command[event.command_name]
+              title << " #{target}"
+            end
+
+            payload = {}
+
+            # Ruby Hashes are ordered based on insertion so do the most important ones first
+
+            add_value('key'.freeze, command, payload)
+            add_bound('query'.freeze, command, payload)
+            add_bound('filter'.freeze, command, payload)
+            add_value('sort'.freeze, command, payload)
+
+            if event.command_name == :findandmodify
+              add_bound('update'.freeze, command, payload)
+            end
+
+            add_value('remove'.freeze, command, payload)
+            add_value('new'.freeze, command, payload)
+
+            if updates = command['updates'.freeze]
+              # AFAICT the gem generally just sends one item in the updates array
+              update = updates[0]
+              update_payload = {}
+              add_bound('q'.freeze, update, update_payload)
+              add_bound('u'.freeze, update, update_payload)
+              add_value('multi'.freeze, update, update_payload)
+              add_value('upsert'.freeze, update, update_payload)
+
+              payload['updates'.freeze] = [update_payload]
+
+              if updates.length > 1
+                payload['updates'.freeze] << '...'
+              end
+            end
+
+            if deletes = command['deletes'.freeze]
+              # AFAICT the gem generally just sends one item in the updates array
+              delete = deletes[0]
+              delete_payload = {}
+              add_bound('q'.freeze, delete, delete_payload)
+              add_value('limit'.freeze, delete, delete_payload)
+
+              payload['deletes'.freeze] = [delete_payload]
+
+              if deletes.length > 1
+                payload['deletes'.freeze] << '...'
+              end
+            end
+
+
+            # We're ignoring documents from insert because they could have completely inconsistent
+            # format which would make it hard to merge.
+
+            opts = {
+              category:    CAT,
+              title:       title,
+              description: payload.empty? ? nil : payload.to_json,
+              meta:        { database: event.database_name }
+            }
+
+            @events[event.operation_id] = Skylight::Core::Fanout.instrument(opts)
+          rescue Exception => e
+            error "failed to begin instrumentation for Mongo; msg=%s", e.message
+          end
+
+          def end_instrumentation(event)
+            if original_event = @events.delete(event.operation_id)
+              meta = {}
+              if event.is_a?(::Mongo::Monitoring::Event::CommandFailed)
+                meta[:exception] = ["CommandFailed", event.message]
+              end
+              Skylight::Core::Fanout.done(original_event, meta)
+            end
+          rescue Exception => e
+            error "failed to end instrumentation for Mongo; msg=%s", e.message
+          end
+
+          def add_value(key, command, payload)
+            if command.has_key?(key)
+              value = command[key]
+              payload[key] = value
+            end
+          end
+
+          def add_bound(key, command, payload)
+            if value = command[key]
+              payload[key] = extract_binds(value)
+            end
+          end
+
+          def extract_binds(hash)
+            ret = {}
+
+            hash.each do |k,v|
+              if v.is_a?(Hash)
+                ret[k] = extract_binds(v)
+              else
+                ret[k] = '?'.freeze
+              end
+            end
+
+            ret
+          end
+
+      end
+    end
+
+    register("Mongo", "mongo", Mongo::Probe.new)
+  end
+end

data/lib/skylight/core/probes/mongoid.rb

@@ -0,0 +1,21 @@
+module Skylight::Core
+  module Probes
+    module Mongoid
+      class Probe
+
+        def install
+          require 'mongoid/version'
+          version = Gem::Version.new(::Mongoid::VERSION)
+
+          if version < Gem::Version.new("5.0")
+            require 'skylight/core/probes/moped'
+          else
+            require 'skylight/core/probes/mongo'
+          end
+        end
+      end
+    end
+
+    register("Mongoid", "mongoid", Mongoid::Probe.new)
+  end
+end

data/lib/skylight/core/probes/moped.rb

@@ -0,0 +1,39 @@
+module Skylight::Core
+  module Probes
+    module Moped
+      class Probe
+
+        def install
+          unless defined?(::Moped::Instrumentable)
+            # Using $stderr here isn't great, but we don't have a logger accessible
+            $stderr.puts "[SKYLIGHT] [#{Skylight::Core::VERSION}] The installed version of Moped doesn't " \
+                          "support instrumentation. The Moped probe will be disabled."
+
+            return
+          end
+
+          ::Moped::Instrumentable.module_eval do
+            alias instrument_without_sk instrument
+
+            def instrument(*args, &block)
+              # Mongoid sets the instrumenter to AS::N
+              if instrumenter == ActiveSupport::Notifications
+                asn_block = block
+              else
+                # If the instrumenter hasn't been changed to AS::N use both
+                asn_block = Proc.new do
+                  ActiveSupport::Notifications.instrument(*args, &block)
+                end
+              end
+
+              instrument_without_sk(*args, &asn_block)
+            end
+          end
+        end
+
+      end
+    end
+
+    register("Moped", "moped", Moped::Probe.new)
+  end
+end

data/lib/skylight/core/probes/net_http.rb

@@ -0,0 +1,64 @@
+require 'skylight/core/formatters/http'
+
+module Skylight::Core
+  module Probes
+    module NetHTTP
+      # Probe for instrumenting Net::HTTP requests. Works by monkeypatching the default Net::HTTP#request method.
+      class Probe
+        DISABLED_KEY = :__skylight_net_http_disabled
+
+        def self.disable
+          Thread.current[DISABLED_KEY] = true
+          yield
+        ensure
+          Thread.current[DISABLED_KEY] = false
+        end
+
+        def self.disabled?
+          !!Thread.current[DISABLED_KEY]
+        end
+
+        def install
+          Net::HTTP.class_eval do
+            alias request_without_sk request
+
+            def request(req, body = nil, &block)
+              if !started? || Probes::NetHTTP::Probe.disabled?
+                return request_without_sk(req, body, &block)
+              end
+
+              method = req.method
+
+              # req['host'] also includes special handling for default ports
+              host, port = req['host'] ? req['host'].split(':') : nil
+
+              # If we're connected with a persistent socket
+              host ||= self.address
+              port ||= self.port
+
+              path   = req.path
+              scheme = use_ssl? ? "https" : "http"
+
+              # Contained in the path
+              query  = nil
+
+              opts = Formatters::HTTP.build_opts(method, scheme, host, port, path, query)
+
+              Skylight::Core::Fanout.instrument(opts) do |spans|
+                spans.each do |(instrumentable, span)|
+                  # TODO: Should we make something more generic?
+                  if (header = instrumentable.correlation_header)
+                    req[header] = instrumentable.span_correlation_header(span)
+                  end
+                end
+                request_without_sk(req, body, &block)
+              end
+            end
+          end
+        end
+      end
+    end
+
+    register("Net::HTTP", "net/http", NetHTTP::Probe.new)
+  end
+end

data/lib/skylight/core/probes/redis.rb

@@ -0,0 +1,71 @@
+module Skylight::Core
+  module Probes
+    module Redis
+      class Probe
+        def install
+          version = defined?(::Redis::VERSION) ? Gem::Version.new(::Redis::VERSION) : nil
+
+          if !version || version < Gem::Version.new("3.0.0")
+            # Using $stderr here isn't great, but we don't have a logger accessible
+            $stderr.puts "[SKYLIGHT] [#{Skylight::Core::VERSION}] The installed version of Redis doesn't " \
+                          "support Middlewares. At least version 3.0.0 is required."
+            return
+          end
+
+          ::Redis::Client.class_eval do
+            alias call_without_sk call
+
+            def call(command, &block)
+              command_name = command[0]
+
+              return call_without_sk(command, &block) if command_name == :auth
+
+              opts = {
+                category: "db.redis.command",
+                title:    command_name.upcase.to_s
+              }
+
+              Skylight::Core::Fanout.instrument(opts) do
+                call_without_sk(command, &block)
+              end
+            end
+          end
+        end
+
+        # Unfortunately, because of the nature of pipelining, there's no way for us to
+        # give a time breakdown on the individual items.
+
+        PIPELINED_OPTS = {
+          category: "db.redis.pipelined".freeze,
+          title:    "PIPELINE".freeze
+        }.freeze
+
+        MULTI_OPTS = {
+          category: "db.redis.multi".freeze,
+          title:    "MULTI".freeze
+        }.freeze
+
+        ::Redis.class_eval do
+          alias pipelined_without_sk pipelined
+
+          def pipelined(&block)
+            Skylight::Core::Fanout.instrument(PIPELINED_OPTS) do
+              pipelined_without_sk(&block)
+            end
+          end
+
+
+          alias multi_without_sk multi
+
+          def multi(&block)
+            Skylight::Core::Fanout.instrument(MULTI_OPTS) do
+              multi_without_sk(&block)
+            end
+          end
+        end
+      end
+    end
+
+    register("Redis", "redis", Redis::Probe.new)
+  end
+end

data/lib/skylight/core/probes/sequel.rb

@@ -0,0 +1,33 @@
+# Supports 3.12.0+
+module Skylight::Core
+  module Probes
+    module Sequel
+      class Probe
+        def install
+          require 'sequel/database/logging'
+
+          method_name = ::Sequel::Database.method_defined?(:log_connection_yield) ? 'log_connection_yield' : 'log_yield'
+
+          ::Sequel::Database.class_eval <<-end_eval
+            alias #{method_name}_without_sk #{method_name}
+
+            def #{method_name}(sql, *args, &block)
+              #{method_name}_without_sk(sql, *args) do
+                ::ActiveSupport::Notifications.instrument(
+                  "sql.sequel",
+                  sql: sql,
+                  name: "SQL",
+                  binds: args
+                ) do
+                  block.call
+                end
+              end
+            end
+          end_eval
+        end
+      end
+    end
+
+    register("Sequel", "sequel", Sequel::Probe.new)
+  end
+end

data/lib/skylight/core/probes/sinatra.rb

@@ -0,0 +1,69 @@
+module Skylight::Core
+  module Probes
+    module Sinatra
+      class Probe
+        def install
+          if ::Sinatra::VERSION < '1.4.0'
+            # Using $stderr here isn't great, but we don't have a logger accessible
+            $stderr.puts "[SKYLIGHT] [#{Skylight::VERSION}] Sinatra must be version 1.4.0 or greater."
+            return
+          end
+
+          class << ::Sinatra::Base
+            alias compile_without_sk! compile!
+
+            def compile!(verb, path, *args, &block)
+              compile_without_sk!(verb, path, *args, &block).tap do |_, _, keys_or_wrapper, wrapper|
+                wrapper ||= keys_or_wrapper
+
+                # Deal with the situation where the path is a regex, and the default behavior
+                # of Ruby stringification produces an unreadable mess
+                if path.is_a?(Regexp)
+                  human_readable = "<sk-regex>%r{#{path.source}}</sk-regex>"
+                  wrapper.instance_variable_set(:@route_name, "#{verb} #{human_readable}")
+                else
+                  wrapper.instance_variable_set(:@route_name, "#{verb} #{path}")
+                end
+              end
+            end
+          end
+
+          ::Sinatra::Base.class_eval do
+            alias dispatch_without_sk! dispatch!
+            alias compile_template_without_sk compile_template
+
+            def dispatch!(*args, &block)
+              dispatch_without_sk!(*args, &block).tap do
+                Skylight::Core::Fanout.registered.each do |target|
+                  instrumenter = target.instrumenter
+                  next unless instrumenter
+                  trace = instrumenter.current_trace
+                  next unless trace
+
+                  # Set the endpoint name to the route name
+                  route = env['sinatra.route']
+                  trace.endpoint = route if route
+                end
+              end
+            end
+
+            def compile_template(engine, data, options, *args, &block)
+              # Pass along a useful "virtual path" to Tilt. The Tilt probe will handle
+              # instrumenting correctly.
+              case data
+              when Symbol
+                options[:sky_virtual_path] = data.to_s
+              else
+                options[:sky_virtual_path] = "Inline template (#{engine})"
+              end
+
+              compile_template_without_sk(engine, data, options, *args, &block)
+            end
+          end
+        end
+      end
+    end
+
+    register("Sinatra::Base", "sinatra/base", Sinatra::Probe.new)
+  end
+end

data/lib/skylight/core/probes/tilt.rb

@@ -0,0 +1,27 @@
+# Should support 0.2+, though not tested against older versions
+module Skylight::Core
+  module Probes
+    module Tilt
+      class Probe
+        def install
+          ::Tilt::Template.class_eval do
+            alias render_without_sk render
+
+            def render(*args, &block)
+              opts = {
+                category: "view.render.template",
+                title: options[:sky_virtual_path] || "Unknown template name"
+              }
+
+              Skylight::Core::Fanout.instrument(opts) do
+                render_without_sk(*args, &block)
+              end
+            end
+          end
+        end
+      end
+    end
+
+    register("Tilt::Template", "tilt/template", Tilt::Probe.new)
+  end
+end

data/lib/skylight/core/probes.rb

@@ -0,0 +1,129 @@
+module Skylight::Core
+  # @api private
+  module Probes
+
+    @@available = nil
+
+    def self.available
+      unless @@available
+        root = File.expand_path("../probes", __FILE__)
+        @@available = {}
+        Dir["#{root}/*.rb"].each do |f|
+          name = File.basename(f, '.rb')
+          @@available[name] = "skylight/core/probes/#{name}"
+        end
+      end
+      @@available
+    end
+
+    def self.probe(*probes)
+      unknown = probes.map(&:to_s) - available.keys
+      unless unknown.empty?
+        raise ArgumentError, "unknown probes: #{unknown.join(', ')}"
+      end
+
+      probes.each do |p|
+        require available[p.to_s]
+      end
+    end
+
+    class ProbeRegistration
+      attr_reader :klass_name, :require_paths, :probe
+
+      def initialize(klass_name, require_paths, probe)
+        @klass_name = klass_name
+        @require_paths = Array(require_paths)
+        @probe = probe
+      end
+
+      def install
+        probe.install
+      end
+    end
+
+    def self.require_hooks
+      @require_hooks ||= {}
+    end
+
+    def self.installed
+      @installed ||= {}
+    end
+
+    def self.is_available?(klass_name)
+      !!Util::Inflector.safe_constantize(klass_name)
+    end
+
+    def self.register(*args)
+      registration = ProbeRegistration.new(*args)
+
+      if is_available?(registration.klass_name)
+        installed[registration.klass_name] ||= []
+        installed[registration.klass_name] << registration
+        registration.install
+      else
+        register_require_hook(registration)
+      end
+    end
+
+    def self.require_hook(require_path)
+      registrations = lookup_by_require_path(require_path)
+      return unless registrations
+
+      registrations.each do |registration|
+        # Double check constant is available
+        if is_available?(registration.klass_name)
+          installed[registration.klass_name] ||= []
+          installed[registration.klass_name] << registration
+          registration.install
+
+          # Don't need this to be called again
+          unregister_require_hook(registration)
+        end
+      end
+    end
+
+    def self.register_require_hook(registration)
+      registration.require_paths.each do |p|
+        require_hooks[p] ||= []
+        require_hooks[p] << registration
+      end
+    end
+
+    def self.unregister_require_hook(registration)
+      registration.require_paths.each do |p|
+        require_hooks[p].delete(registration)
+        require_hooks.delete(p) if require_hook[p].empty?
+      end
+    end
+
+    def self.lookup_by_require_path(require_path)
+      require_hooks[require_path]
+    end
+  end
+end
+
+# Allow hooking require
+# @api private
+module ::Kernel
+  alias require_without_sk require
+
+  def require(name)
+    ret = require_without_sk(name)
+
+    begin
+      Skylight::Core::Probes.require_hook(name)
+    rescue Exception
+      # FIXME: Log these errors
+    end
+
+    ret
+  rescue LoadError
+    # Support pre-2.0 style requires
+    if name =~ %r{^skylight/probes/(.+)}
+      warn "[DEPRECATION] Requiring Skylight probes by path is deprecated. Use `Skylight.probe(:#{$1})` instead."
+      require "skylight/core/probes/#{$1}"
+    else
+      raise
+    end
+  end
+end