skylight-core 2.0.0.beta1

data/lib/skylight/core/normalizers/sql.rb

@@ -0,0 +1,49 @@
+require "json"
+
+module Skylight::Core
+  module Normalizers
+    # Normalizer for SQL requests
+    class SQL < Normalizer
+      CAT = "db.sql.query".freeze
+
+      # @param trace [Skylight::Messages::Trace::Builder] ignored, only present to match API
+      # @param name [String] ignored, only present to match API
+      # @param payload [Hash]
+      # @option payload [String] [:name] The SQL operation
+      # @option payload [Hash] [:binds] The bound parameters
+      # @return [Array]
+      def normalize(trace, name, payload)
+        case payload[:name]
+        when "SCHEMA".freeze, "CACHE".freeze
+          return :skip
+        else
+          name  = CAT
+          title = payload[:name] || "SQL".freeze
+        end
+
+        binds = payload[:binds]
+
+        if binds && !binds.empty?
+          binds = binds.map { |_col, val| val.inspect }
+        end
+
+        begin
+          extracted_title, sql = extract_binds(trace.instrumenter, payload, binds)
+          [ name, extracted_title || title, sql ]
+        rescue => e
+          # FIXME: Rust errors get written to STDERR and don't come through here
+          if config[:log_sql_parse_errors]
+            config.logger.warn "failed to extract binds in SQL; sql=#{payload[:sql].inspect}; exception=#{e.inspect}"
+          end
+          [ name, title, nil ]
+        end
+      end
+
+      private
+
+      def extract_binds(instrumenter, payload, _precalculated)
+        instrumenter.process_sql(payload[:sql])
+      end
+    end
+  end
+end

data/lib/skylight/core/normalizers.rb

@@ -0,0 +1,170 @@
+require 'skylight/core/normalizers/default'
+
+module Skylight::Core
+  # @api private
+  # Convert AS::N events to Skylight events
+  module Normalizers
+
+    DEFAULT = Default.new
+
+    def self.register(name, klass)
+      (@registry ||= {})[name] = klass
+      klass
+    end
+
+    def self.build(config)
+      normalizers = {}
+
+      (@registry || {}).each do |k, klass|
+        unless klass.method_defined?(:normalize)
+          # TODO: Warn
+          next
+        end
+
+        normalizers[k] = klass.new(config)
+      end
+
+      Container.new(normalizers)
+    end
+
+    class Normalizer
+      def self.register(name)
+        Normalizers.register(name, self)
+      end
+
+      attr_reader :config
+
+      def initialize(config)
+        @config = config
+        setup if respond_to?(:setup)
+      end
+
+      def normalize(trace, name, payload)
+        :skip
+      end
+
+      def normalize_after(trace, span, name, payload)
+      end
+    end
+
+    # Base Normalizer for Rails rendering
+    class RenderNormalizer < Normalizer
+      include Util::AllocationFree
+
+      def setup
+        @paths = config['normalizers.render.view_paths'] || []
+      end
+
+      # Generic normalizer for renders
+      # @param category [String]
+      # @param payload [Hash]
+      # @option payload [String] :identifier
+      # @return [Array]
+      def normalize_render(category, payload)
+        if path = payload[:identifier]
+          title = relative_path(path)
+          path = nil if path == title
+        end
+
+        [ category, title, nil ]
+      end
+
+      def relative_path(path)
+        return path if relative_path?(path)
+
+        root = array_find(@paths) { |p| path.start_with?(p) }
+        type = :project
+
+        unless root
+          root = array_find(Gem.path) { |p| path.start_with?(p) }
+          type = :gem
+        end
+
+        if root
+          start = root.size
+          start += 1 if path.getbyte(start) == SEPARATOR_BYTE
+          if type == :gem
+            "$GEM_PATH/#{path[start, path.size]}"
+          else
+            path[start, path.size]
+          end
+        else
+          "Absolute Path"
+        end
+      end
+
+    private
+      def relative_path?(path)
+        !absolute_path?(path)
+      end
+
+      SEPARATOR_BYTE = File::SEPARATOR.ord
+
+      if File.const_defined?(:NULL) ? File::NULL == "NUL" : RbConfig::CONFIG['host_os'] =~ /mingw|mswin32/
+        # This is a DOSish environment
+        ALT_SEPARATOR_BYTE = File::ALT_SEPARATOR && File::ALT_SEPARATOR.ord
+        COLON_BYTE = ":".ord
+        def absolute_path?(path)
+          if alpha?(path.getbyte(0)) && path.getbyte(1) == COLON_BYTE
+            byte2 = path.getbyte(2)
+            byte2 == SEPARATOR_BYTE || byte2 == ALT_SEPARATOR_BYTE
+          end
+        end
+
+        def alpha?(byte)
+          byte >= 65 and byte <= 90 || byte >= 97 and byte <= 122
+        end
+      else
+        def absolute_path?(path)
+          path.getbyte(0) == SEPARATOR_BYTE
+        end
+      end
+    end
+
+    class Container
+      def initialize(normalizers)
+        @normalizers = normalizers
+      end
+
+      def keys
+        @normalizers.keys
+      end
+
+      def normalize(trace, name, payload)
+        normalizer_for(name).normalize(trace, name, payload)
+      end
+
+      def normalize_after(trace, span, name, payload)
+        normalizer_for(name).normalize_after(trace, span, name, payload)
+      end
+
+      def normalizer_for(name)
+        @normalizers[name] || DEFAULT
+      end
+    end
+
+    %w( action_controller/process_action
+        action_controller/send_file
+        action_view/render_collection
+        action_view/render_partial
+        action_view/render_template
+        active_model_serializers/render
+        active_record/instantiation
+        active_record/sql
+        active_support/cache
+        coach/handler_finish
+        coach/middleware_finish
+        couch_potato/query
+        data_mapper/sql
+        elasticsearch/request
+        faraday/request
+        grape/endpoint
+        moped/query
+        sequel/sql).each do |file|
+      require "skylight/core/normalizers/#{file}"
+    end
+
+    # The following are not required by default as they are of dubious usefulness:
+    # - active_job/enqueue_at
+  end
+end

data/lib/skylight/core/probes/action_controller.rb

@@ -0,0 +1,31 @@
+module Skylight::Core
+  module Probes
+    module ActionController
+      class Probe
+        def install
+          ::ActionController::Instrumentation.class_eval do
+            private
+            alias append_info_to_payload_without_sk append_info_to_payload
+            def append_info_to_payload(payload)
+              append_info_to_payload_without_sk(payload)
+              rendered_mime = begin
+                if respond_to?(:rendered_format)
+                  rendered_format
+                elsif content_type.is_a?(Mime::Type)
+                  content_type
+                elsif content_type.respond_to?(:to_s)
+                  type_str = content_type.to_s.split(';').first
+                  Mime::Type.lookup(type_str) unless type_str.blank?
+                end
+              end
+              payload[:rendered_format] = rendered_mime.try(:ref)
+              payload[:variant] = request.respond_to?(:variant) ? request.variant : nil
+            end
+          end
+        end
+      end
+    end
+
+    register("ActionController::Instrumentation", "action_controller/metal/instrumentation", ActionController::Probe.new)
+  end
+end

data/lib/skylight/core/probes/action_view.rb

@@ -0,0 +1,37 @@
+module Skylight::Core
+  module Probes
+    module ActionView
+      class Probe
+        def install
+          ::ActionView::TemplateRenderer.class_eval do
+            alias render_with_layout_without_sk render_with_layout
+
+            def render_with_layout(path, locals, *args, &block) #:nodoc:
+              layout = nil
+
+              if path
+                if method(:find_layout).arity == 3
+                  # Rails 5
+                  layout = find_layout(path, locals.keys, [formats.first])
+                else
+                  # Rails 4
+                  layout = find_layout(path, locals.keys)
+                end
+              end
+
+              if layout
+                instrument(:template, :identifier => layout.identifier) do
+                  render_with_layout_without_sk(path, locals, *args, &block)
+                end
+              else
+                render_with_layout_without_sk(path, locals, *args, &block)
+              end
+            end
+          end
+        end
+      end
+    end
+
+    register("ActionView::TemplateRenderer", "action_view", ActionView::Probe.new)
+  end
+end

data/lib/skylight/core/probes/active_model_serializers.rb

@@ -0,0 +1,55 @@
+module Skylight::Core
+  module Probes
+    module ActiveModelSerializers
+      class Probe
+        def install
+          version = nil
+
+          # File moved location between version
+          %w(serializer serializers).each do |dir|
+            begin
+              require "active_model/#{dir}/version"
+            rescue LoadError
+            end
+          end
+
+          if defined?(::ActiveModel::Serializer::VERSION)
+            version = Gem::Version.new(::ActiveModel::Serializer::VERSION)
+          end
+
+          if !version || version < Gem::Version.new("0.5.0")
+            # Using $stderr here isn't great, but we don't have a logger accessible
+            $stderr.puts "[SKYLIGHT] [#{Skylight::Core::VERSION}] Instrumention is only available for " \
+                          "ActiveModelSerializers version 0.5.0 and greater."
+            return
+          end
+
+          # We don't actually support the RCs correctly, requires
+          # a release after 0.10.0.rc3
+          if version >= Gem::Version.new("0.10.0.rc1")
+            # AS::N is built in to newer versions
+            return
+          end
+
+          # End users could override as_json without calling super, but it's likely safer
+          # than overriding serializable_array/hash/object.
+
+          [::ActiveModel::Serializer, ::ActiveModel::ArraySerializer].each do |klass|
+            klass.class_eval do
+              alias as_json_without_sk as_json
+              def as_json(*args)
+                payload = { serializer: self.class }
+                ActiveSupport::Notifications.instrument('render.active_model_serializers', payload) do
+                  as_json_without_sk(*args)
+                end
+              end
+            end
+          end
+
+        end
+      end
+    end
+
+    register("ActiveModel::Serializer", "active_model/serializer", ActiveModelSerializers::Probe.new)
+  end
+end

data/lib/skylight/core/probes/elasticsearch.rb

@@ -0,0 +1,37 @@
+module Skylight::Core
+  module Probes
+    module Elasticsearch
+      class Probe
+        def install
+          ::Elasticsearch::Transport::Transport::Base.class_eval do
+            alias perform_request_without_sk perform_request
+            def perform_request(method, path, *args, &block)
+              ActiveSupport::Notifications.instrument "request.elasticsearch",
+                                                      name:   'Request',
+                                                      method: method,
+                                                      path:   path do
+
+
+                # Prevent HTTP-related probes from firing
+                Skylight::Core::Normalizers::Faraday::Request.disable do
+                  disable_skylight_probe(:NetHTTP) do
+                    disable_skylight_probe(:HTTPClient) do
+                      perform_request_without_sk(method, path, *args, &block)
+                    end
+                  end
+                end
+              end
+            end
+
+            def disable_skylight_probe(class_name, &block)
+              klass = Probes.const_get(class_name).const_get(:Probe) rescue nil
+              klass ? klass.disable(&block) : block.call
+            end
+          end
+        end
+      end
+    end
+
+    register("Elasticsearch", "elasticsearch", Elasticsearch::Probe.new)
+  end
+end

data/lib/skylight/core/probes/excon/middleware.rb

@@ -0,0 +1,72 @@
+require 'skylight/core/formatters/http'
+
+module Skylight::Core
+  module Probes
+    module Excon
+
+      # Middleware for Excon that instruments requests
+      class Middleware < ::Excon::Middleware::Base
+
+        # This probably won't work since config isn't defined
+        include Util::Logging
+
+        def initialize(*)
+          @requests = {}
+          super
+        end
+
+        # TODO:
+        # - Consider whether a LIFO queue would be sufficient
+        # - Check that errors can't be called without a request
+
+        def request_call(datum)
+          begin_instrumentation(datum)
+          super
+        end
+
+        def response_call(datum)
+          super
+        ensure
+          end_instrumentation(datum)
+        end
+
+        def error_call(datum)
+          super
+        ensure
+          end_instrumentation(datum)
+        end
+
+        private
+
+          def begin_instrumentation(datum)
+            method = datum[:method].to_s
+            scheme = datum[:scheme]
+            host   = datum[:host]
+            # TODO: Maybe don't show other default ports like 443
+            port   = datum[:port] != 80 ? datum[:port] : nil
+            path   = datum[:path]
+            query  = datum[:query]
+
+            opts = Formatters::HTTP.build_opts(method, scheme, host, port, path, query)
+
+            @requests[datum.object_id] = Skylight::Core::Fanout.instrument(opts)
+          rescue Exception => e
+            error "failed to begin instrumentation for Excon; msg=%s", e.message
+          end
+
+          def end_instrumentation(datum)
+            if request = @requests.delete(datum.object_id)
+              meta = { }
+              if datum[:error].is_a?(Exception)
+                meta[:exception_object] = datum[:error]
+              end
+              Skylight::Core::Fanout.done(request, meta)
+            end
+          rescue Exception => e
+            error "failed to end instrumentation for Excon; msg=%s", e.message
+          end
+
+      end
+    end
+  end
+end

data/lib/skylight/core/probes/excon.rb

@@ -0,0 +1,26 @@
+module Skylight::Core
+  module Probes
+    module Excon
+      # Probe for instrumenting Excon requests. Installs {Excon::Middleware} to achieve this.
+      class Probe
+        def install
+          if defined?(::Excon::Middleware)
+            # Don't require until installation since it depends on Excon being loaded
+            require 'skylight/core/probes/excon/middleware'
+
+            idx = ::Excon.defaults[:middlewares].index(::Excon::Middleware::Instrumentor)
+
+            # TODO: Handle possibility of idx being nil
+            ::Excon.defaults[:middlewares].insert(idx, Probes::Excon::Middleware)
+          else
+            # Using $stderr here isn't great, but we don't have a logger accessible
+            $stderr.puts "[SKYLIGHT] [#{Skylight::Core::VERSION}] The installed version of Excon doesn't " \
+                          "support Middlewares. The Excon probe will be disabled."
+          end
+        end
+      end
+    end
+
+    register("Excon", "excon", Excon::Probe.new)
+  end
+end

data/lib/skylight/core/probes/faraday.rb

@@ -0,0 +1,22 @@
+module Skylight::Core
+  module Probes
+    module Faraday
+      class Probe
+        def install
+          ::Faraday::Connection.class_eval do
+            alias initialize_without_sk initialize
+
+            def initialize(*args, &block)
+              initialize_without_sk(*args, &block)
+
+              @builder.insert 0, ::Faraday::Request::Instrumentation
+            end
+          end
+        end
+
+      end
+    end
+
+    register("Faraday", "faraday", Faraday::Probe.new)
+  end
+end

data/lib/skylight/core/probes/grape.rb

@@ -0,0 +1,80 @@
+module Skylight::Core
+  module Probes
+    module Grape
+      class Probe
+        def install
+          version = Gem::Version.new(::Grape::VERSION)
+
+          if version > Gem::Version.new("0.12.1")
+            # AS::N is built in to newer versions
+            return
+          end
+
+          if version < Gem::Version.new("0.10.0")
+            # Using $stderr here isn't great, but we don't have a logger accessible
+            $stderr.puts "[SKYLIGHT] [#{Skylight::Core::VERSION}] The Grape probe only works with version 0.10.0+ " \
+                          "and will be disabled."
+
+            return
+          end
+
+          ::Grape::Endpoint.class_eval do
+            alias initialize_without_sk initialize
+            def initialize(*args, &block)
+              initialize_without_sk(*args, &block)
+
+              # This solution of wrapping the block is effective, but potentially fragile.
+              # A cleaner solution would be to call the original initialize with the already
+              # modified block. However, Grape does some odd stuff with the block binding
+              # that makes this difficult to reason about.
+              if original_block = @block
+                @block = lambda do |endpoint_instance|
+                  ActiveSupport::Notifications.instrument('endpoint_render.grape', endpoint: endpoint_instance) do
+                    original_block.call(endpoint_instance)
+                  end
+                end
+              end
+            end
+
+            alias run_without_sk run
+            def run(*args)
+              ActiveSupport::Notifications.instrument('endpoint_run.grape', endpoint: self) do
+                run_without_sk(*args)
+              end
+            end
+
+            alias run_filters_without_sk run_filters
+            def run_filters(filters)
+              # Unfortunately, the type isn't provided to the method so we have
+              # to try to guess it by looking at the contents. This is only reliable
+              # if the filters aren't empty.
+              if filters && !filters.empty?
+                type = case filters
+                  when befores            then :before
+                  when before_validations then :before_validation
+                  when after_validations  then :after_validation
+                  when afters             then :after
+                  else                         :other
+                  end
+              else
+                type = :unknown
+              end
+
+              payload = {
+                endpoint: self,
+                filters: filters,
+                type: type
+              }
+
+              ActiveSupport::Notifications.instrument('endpoint_run_filters.grape', payload) do
+                run_filters_without_sk(filters)
+              end
+            end
+          end
+        end
+      end
+    end
+
+    register("Grape::Endpoint", "grape/endpoint", Grape::Probe.new)
+  end
+end

data/lib/skylight/core/probes/httpclient.rb

@@ -0,0 +1,46 @@
+require 'skylight/core/formatters/http'
+
+module Skylight::Core
+  module Probes
+    module HTTPClient
+      class Probe
+        DISABLED_KEY = :__skylight_httpclient_disabled
+
+        def self.disable
+          Thread.current[DISABLED_KEY] = true
+          yield
+        ensure
+          Thread.current[DISABLED_KEY] = false
+        end
+
+        def self.disabled?
+          !!Thread.current[DISABLED_KEY]
+        end
+
+        def install
+          ::HTTPClient.class_eval do
+            # HTTPClient has request methods on the class object itself,
+            # but the internally instantiate a client and perform the method
+            # on that, so this instance method override will cover both
+            # `HTTPClient.get(...)` and `HTTPClient.new.get(...)`
+
+            alias do_request_without_sk do_request
+            def do_request(method, uri, query, body, header, &block)
+              if Probes::HTTPClient::Probe.disabled?
+                return do_request_without_sk(method, uri, query, body, header, &block)
+              end
+
+              opts = Formatters::HTTP.build_opts(method, uri.scheme, uri.host, uri.port, uri.path, uri.query)
+
+              Skylight::Core::Fanout.instrument(opts) do
+                do_request_without_sk(method, uri, query, body, header, &block)
+              end
+            end
+          end
+        end
+      end # class Probe
+    end # module Probes::HTTPClient
+
+    register("HTTPClient", "httpclient", HTTPClient::Probe.new)
+  end
+end

data/lib/skylight/core/probes/middleware.rb

@@ -0,0 +1,58 @@
+module Skylight::Core
+  module Probes
+    module Middleware
+      class Probe
+        def install
+          ActionDispatch::MiddlewareStack::Middleware.class_eval do
+            alias build_without_sk build
+            def build(*args)
+              sk_instrument_middleware(build_without_sk(*args))
+            end
+
+            def sk_instrument_middleware(middleware)
+              return middleware if middleware.is_a?(Skylight::Core::Middleware)
+
+              # Not sure how this would actually happen
+              return middleware if middleware.respond_to?(:call_without_sk)
+
+              # On Rails 3, ActionDispatch::Session::CookieStore is frozen, for one
+              return middleware if middleware.frozen?
+
+              middleware.instance_eval do
+                alias call_without_sk call
+                def call(*args, &block)
+                  traces = Skylight::Core::Fanout.registered.map do |r|
+                    r.instrumenter ? r.instrumenter.current_trace : nil
+                  end.compact
+
+                  return call_without_sk(*args, &block) if traces.empty?
+
+                  begin
+                    name = self.class.name || "Anonymous Middleware"
+
+                    traces.each{|t| t.endpoint = name }
+
+                    spans = Skylight::Core::Fanout.instrument(title: name, category: "rack.middleware")
+                    resp = call_without_sk(*args, &block)
+
+                    Skylight::Core::Middleware.with_after_close(resp) do
+                      Skylight::Core::Fanout.done(spans)
+                    end
+                  rescue Exception => e
+                    # FIXME: Log this?
+                    Skylight::Core::Fanout.done(spans, exception_object: e)
+                    raise
+                  end
+                end
+              end
+
+              middleware
+            end
+          end
+        end
+      end
+    end
+
+    register("ActionDispatch::MiddlewareStack::Middleware", "actionpack/action_dispatch", Middleware::Probe.new)
+  end
+end