skylight-core 2.0.0.beta1

data/lib/skylight/core/instrumentable.rb

@@ -0,0 +1,144 @@
+module Skylight
+  module Core
+    module Instrumentable
+
+      def self.included(base)
+        base.extend(ClassMethods)
+
+        base.const_set(:LOCK, Mutex.new)
+
+        base.class_eval do
+          at_exit do
+            if RUBY_VERSION == '1.9.2'
+              # workaround for MRI bug losing exit status in at_exit block
+              # http://bugs.ruby-lang.org/issues/5218
+              exit_status = $!.status if $!.is_a?(SystemExit)
+              stop!
+              exit exit_status if exit_status
+            else
+              stop!
+            end
+          end
+        end
+
+        Skylight::Core::Fanout.register(base)
+      end
+
+      module ClassMethods
+
+        def instrumenter_class
+          Skylight::Core::Instrumenter
+        end
+
+        def instrumenter
+          @instrumenter
+        end
+
+        def correlation_header
+          nil
+        end
+
+        # Start instrumenting
+        def start!(config=nil)
+          return @instrumenter if @instrumenter
+
+          const_get(:LOCK).synchronize do
+            return @instrumenter if @instrumenter
+
+            config ||= {}
+            config = config_class.load(config) unless config.is_a?(config_class)
+
+            @instrumenter = instrumenter_class.new(config).start!
+          end
+        rescue => e
+          message = sprintf("[SKYLIGHT] [#{VERSION}] Unable to start Instrumenter; msg=%s; class=%s", e.message, e.class)
+          if config && config.respond_to?(:logger)
+            config.logger.warn message
+          else
+            warn message
+          end
+          false
+        end
+
+        # Stop instrumenting
+        def stop!
+          const_get(:LOCK).synchronize do
+            return unless @instrumenter
+            # This is only really helpful for getting specs to pass.
+            @instrumenter.current_trace = nil
+
+            @instrumenter.shutdown
+            @instrumenter = nil
+          end
+        end
+
+        # Check tracing
+        def tracing?
+          instrumenter && instrumenter.current_trace
+        end
+
+        # Start a trace
+        def trace(endpoint=nil, cat=nil, title=nil, meta=nil)
+          unless instrumenter
+            return yield if block_given?
+            return
+          end
+
+          if block_given?
+            instrumenter.trace(endpoint, cat || DEFAULT_CATEGORY, title, nil, meta) { yield }
+          else
+            instrumenter.trace(endpoint, cat || DEFAULT_CATEGORY, title, nil, meta)
+          end
+        end
+
+        # Instrument
+        def instrument(opts = DEFAULT_OPTIONS, &block)
+          unless instrumenter
+            return yield if block_given?
+            return
+          end
+
+          if Hash === opts
+            category    = opts[:category] || DEFAULT_CATEGORY
+            title       = opts[:title]
+            desc        = opts[:description]
+            meta        = opts[:meta]
+            if opts.key?(:annotations)
+              warn "call to #instrument included deprecated annotations"
+            end
+          else
+            category    = DEFAULT_CATEGORY
+            title       = opts.to_s
+            desc        = nil
+            meta        = nil
+          end
+
+          instrumenter.instrument(category, title, desc, meta, &block)
+        end
+
+        def span_correlation_header(span)
+          return unless instrumenter
+          instrumenter.span_correlation_header(span)
+        end
+
+        # End a span
+        def done(span, meta=nil)
+          return unless instrumenter
+          instrumenter.done(span, meta)
+        end
+
+        # Temporarily disable
+        def disable
+          unless instrumenter
+            return yield if block_given?
+            return
+          end
+
+          instrumenter.disable { yield }
+        end
+
+      end
+
+    end
+  end
+end

data/lib/skylight/core/instrumenter.rb

@@ -0,0 +1,249 @@
+require 'thread'
+require 'strscan'
+
+module Skylight::Core
+  # @api private
+  class Instrumenter
+    KEY  = :__skylight_current_trace
+
+    TOO_MANY_UNIQUES = "<too many unique descriptions>"
+
+    include Util::Logging
+
+    class TraceInfo
+      def initialize(key=KEY)
+        @key = key
+      end
+
+      def current
+        Thread.current[@key]
+      end
+
+      def current=(trace)
+        Thread.current[@key] = trace
+      end
+    end
+
+    attr_reader :config, :gc, :trace_info
+
+    def self.trace_class
+      Trace
+    end
+
+    def self.native_new
+      raise "not implemented"
+    end
+
+    def self.new(config)
+      config.validate!
+
+      inst = native_new(config.to_native_env)
+      inst.send(:initialize, config)
+      inst
+    end
+
+    def initialize(config)
+      @gc = config.gc
+      @config = config
+      @subscriber = Subscriber.new(config, self)
+
+      key = "#{KEY}_#{self.class.trace_class.name}".gsub(/\W/, '_')
+      @trace_info = @config[:trace_info] || TraceInfo.new(key)
+    end
+
+    def native_start
+      raise "not implemented"
+    end
+
+    def native_stop
+      raise "not implemented"
+    end
+
+    def native_track_desc
+      raise "not implemented"
+    end
+
+    def native_submit_trace
+      raise "not implemented"
+    end
+
+    def current_trace
+      @trace_info.current
+    end
+
+    def current_trace=(trace)
+      @trace_info.current = trace
+    end
+
+    def check_install!
+      true
+    end
+
+    def start!
+      # We do this here since we can't report these issues via Gem install without stopping install entirely.
+      check_install!
+
+      t { "starting instrumenter" }
+
+      unless config.validate_with_server
+        log_error "invalid config"
+        return
+      end
+
+      t { "starting native instrumenter" }
+      unless native_start
+        warn "failed to start instrumenter"
+        return
+      end
+
+      config.gc.enable
+      @subscriber.register!
+
+      self
+
+    rescue Exception => e
+      log_error "failed to start instrumenter; msg=%s; config=%s", e.message, config.inspect
+      t { e.backtrace.join("\n") }
+      nil
+    end
+
+    def shutdown
+      @subscriber.unregister!
+      native_stop
+    end
+
+    def trace(endpoint, cat, title=nil, desc=nil, meta=nil)
+      # If a trace is already in progress, continue with that one
+      if trace = @trace_info.current
+        return yield(trace) if block_given?
+        return trace
+      end
+
+      begin
+        trace = self.class.trace_class.new(self, endpoint, Util::Clock.nanos, cat, title, desc, meta)
+      rescue Exception => e
+        log_error e.message
+        t { e.backtrace.join("\n") }
+        return
+      end
+
+      @trace_info.current = trace
+      return trace unless block_given?
+
+      begin
+        yield trace
+
+      ensure
+        @trace_info.current = nil
+        t { "submitting trace" }
+        trace.submit
+      end
+    end
+
+    def disable
+      @disabled = true
+      yield
+    ensure
+      @disabled = false
+    end
+
+    def disabled?
+      @disabled
+    end
+
+    def self.match?(string, regex)
+      @scanner ||= StringScanner.new('')
+      @scanner.string = string
+      @scanner.match?(regex)
+    end
+
+    def match?(string, regex)
+      self.class.match?(string, regex)
+    end
+
+    def instrument(cat, title=nil, desc=nil, meta=nil)
+      raise ArgumentError, 'cat is required' unless cat
+
+      unless trace = @trace_info.current
+        return yield if block_given?
+        return
+      end
+
+      cat = cat.to_s
+
+      unless match?(cat, Skylight::CATEGORY_REGEX)
+        warn "invalid skylight instrumentation category; value=%s", cat
+        return yield if block_given?
+        return
+      end
+
+      cat = "other.#{cat}" unless match?(cat, Skylight::TIER_REGEX)
+
+      unless sp = trace.instrument(cat, title, desc, meta)
+        return yield if block_given?
+        return
+      end
+
+      return sp unless block_given?
+
+      meta = {}
+      begin
+        yield sp
+      rescue Exception => e
+        meta = { exception: [e.class.name, e.message], exception_object: e }
+        raise e
+      ensure
+        trace.done(sp, meta)
+      end
+    end
+
+    def span_correlation_header(span)
+      return unless trace = @trace_info.current
+      trace.span_correlation_header(span)
+    end
+
+    def done(span, meta=nil)
+      return unless trace = @trace_info.current
+      trace.done(span, meta)
+    end
+
+    def limited_description(description)
+      endpoint = @trace_info.current.endpoint
+
+      if description
+        if native_track_desc(endpoint, description)
+          description
+        else
+          TOO_MANY_UNIQUES
+        end
+      end
+    end
+
+    def process(trace)
+      t { fmt "processing trace" }
+
+      if ignore?(trace)
+        t { fmt "ignoring trace" }
+        return false
+      end
+
+      begin
+        native_submit_trace(trace)
+        true
+      rescue => e
+        warn "failed to submit trace to worker; err=%s", e
+        false
+      end
+    end
+
+    def ignore?(trace)
+      config.ignored_endpoints.include?(trace.endpoint.sub(%r{<sk-segment>.+</sk-segment>}, ''))
+    end
+
+    # Return [title, sql]
+    def process_sql(sql)
+      [nil, sql]
+    end
+
+  end
+end

data/lib/skylight/core/middleware.rb

@@ -0,0 +1,101 @@
+module Skylight::Core
+  # @api private
+  class Middleware
+
+    class BodyProxy
+      def initialize(body, &block)
+        @body, @block, @closed = body, block, false
+      end
+
+      def respond_to?(*args)
+        return false if args.first.to_s =~ /^to_ary$/
+        super or @body.respond_to?(*args)
+      end
+
+      def close
+        return if @closed
+        @closed = true
+        begin
+          @body.close if @body.respond_to? :close
+        ensure
+          @block.call
+        end
+      end
+
+      def closed?
+        @closed
+      end
+
+      # N.B. This method is a special case to address the bug described by
+      # https://github.com/rack/rack/issues/434.
+      # We are applying this special case for #each only. Future bugs of this
+      # class will be handled by requesting users to patch their ruby
+      # implementation, to save adding too many methods in this class.
+      def each(*args, &block)
+        @body.each(*args, &block)
+      end
+
+      def method_missing(*args, &block)
+        super if args.first.to_s =~ /^to_ary$/
+        @body.__send__(*args, &block)
+      end
+    end
+
+    def self.with_after_close(resp, &block)
+      # Responses should be finished but in some situations they aren't
+      #   e.g. https://github.com/ruby-grape/grape/issues/1041
+      if resp.respond_to?(:finish)
+        resp = resp.finish
+      end
+
+      resp[2] = BodyProxy.new(resp[2], &block)
+      resp
+    end
+
+    include Util::Logging
+
+    # For Util::Logging
+    attr_reader :config
+
+    def initialize(app, opts={})
+      @app = app
+      @config = opts[:config]
+    end
+
+    def instrumentable
+      Skylight
+    end
+
+    # Allow for overwriting
+    def endpoint_name(_env)
+      "Rack"
+    end
+
+    def endpoint_meta(_env)
+      nil
+    end
+
+    def call(env)
+      if env["REQUEST_METHOD"] == "HEAD"
+        t { "middleware skipping HEAD" }
+        @app.call(env)
+      else
+        begin
+          t { "middleware beginning trace" }
+          trace = instrumentable.trace(endpoint_name(env), 'app.rack.request', nil, endpoint_meta(env))
+          resp = @app.call(env)
+
+          if trace
+            Middleware.with_after_close(resp) { trace.submit }
+          else
+            resp
+          end
+        rescue Exception
+          t { "middleware exception: #{trace}"}
+          trace.submit if trace
+          raise
+        end
+      end
+    end
+  end
+end

data/lib/skylight/core/normalizers/action_controller/process_action.rb

@@ -0,0 +1,50 @@
+module Skylight::Core
+  module Normalizers
+    module ActionController
+      # Normalizer for processing a Rails controller action
+      class ProcessAction < Normalizer
+        register "process_action.action_controller"
+
+        CAT = "app.controller.request".freeze
+
+        # Payload Keys: controller, action, params, format, method, path
+        #   Additional keys available in `normalize_after`: status, view_runtime
+        #     Along with ones added by probe: variant
+
+        # @param trace [Skylight::Messages::Trace::Builder]
+        # @param name [String] ignored, only present to match API
+        # @param payload [Hash]
+        # @option payload [String] :controller Controller name
+        # @option payload [String] :action Action name
+        # @return [Array]
+        def normalize(trace, name, payload)
+          trace.endpoint = controller_action(payload)
+          [ CAT, trace.endpoint, nil ]
+        end
+
+        def normalize_after(trace, span, name, payload)
+          return unless config.enable_segments?
+
+          # Show 'error' if there's an unhandled exception or if the status is 4xx or 5xx
+          if payload[:exception] || payload[:exception_object] || payload[:status].to_s =~ /^[45]/
+            segment = "error"
+          # We won't have a rendered_format if it's a `head` outside of a `respond_to` block.
+          elsif payload[:rendered_format]
+            # We only show the variant if we actually have a format
+            segment = [payload[:rendered_format], payload[:variant]].compact.flatten.join('+')
+          end
+
+          if segment
+            trace.endpoint += "<sk-segment>#{segment}</sk-segment>"
+          end
+        end
+
+      private
+
+        def controller_action(payload)
+          "#{payload[:controller]}##{payload[:action]}"
+        end
+      end
+    end
+  end
+end

data/lib/skylight/core/normalizers/action_controller/send_file.rb

@@ -0,0 +1,50 @@
+module Skylight::Core
+  module Normalizers
+    module ActionController
+
+      # Temporary hacks
+      begin
+        require "action_dispatch/http/mime_type"
+        require "action_dispatch/http/mime_types"
+        require "rack/utils"
+
+        class SendFile < Normalizer
+          register "send_file.action_controller"
+
+          CAT = "app.controller.send_file".freeze
+          TITLE = "send file".freeze
+
+          def normalize(trace, name, payload)
+            path = payload[:path]
+
+            title = TITLE
+
+            # depending on normalization, we probably want this to eventually
+            # include the full path, but we need to make sure we have a good
+            # deduping strategy first.
+            desc = nil
+
+            [ CAT, title, desc ]
+          end
+
+        private
+
+          OCTET_STREAM = "application/octet-stream".freeze
+          ATTACHMENT = "attachment".freeze
+
+          def initialize(*)
+            super
+
+            @mimes = Mime::SET.reduce({}) do |hash, mime|
+              hash[mime.symbol] = mime.to_s.dup.freeze
+              hash
+            end
+          end
+        end
+
+      rescue LoadError
+      end
+
+    end
+  end
+end

data/lib/skylight/core/normalizers/action_view/render_collection.rb

@@ -0,0 +1,22 @@
+module Skylight::Core
+  module Normalizers
+    module ActionView
+      # Normalizer for Rails collection rendering
+      class RenderCollection < RenderNormalizer
+        register "render_collection.action_view"
+
+        CAT = "view.render.collection".freeze
+
+        # @param trace [Skylight::Messages::Trace::Builder] ignored, only present to match API
+        # @param name [String] ignored, only present to match API
+        # @param payload (see RenderNormalizer#normalize_render)
+        # @option payload (see RenderNormalizer#normalize_render)
+        # @option payload [Integer] :count
+        # @return (see RenderNormalizer#normalize_render)
+        def normalize(trace, name, payload)
+          normalize_render(CAT, payload)
+        end
+      end
+    end
+  end
+end

data/lib/skylight/core/normalizers/action_view/render_partial.rb

@@ -0,0 +1,21 @@
+module Skylight::Core
+  module Normalizers
+    module ActionView
+      # Normalizer for Rails partial rendering
+      class RenderPartial < RenderNormalizer
+        register "render_partial.action_view"
+
+        CAT = "view.render.template".freeze
+
+        # @param trace [Skylight::Messages::Trace::Builder] ignored, only present to match API
+        # @param name [String] ignored, only present to match API
+        # @param payload (see RenderNormalizer#normalize_render)
+        # @option payload (see RenderNormalizer#normalize_render)
+        # @return (see RenderNormalizer#normalize_render)
+        def normalize(trace, name, payload)
+          normalize_render(CAT, payload)
+        end
+      end
+    end
+  end
+end

data/lib/skylight/core/normalizers/action_view/render_template.rb

@@ -0,0 +1,21 @@
+module Skylight::Core
+  module Normalizers
+    module ActionView
+      # Normalizer for Rails template rendering
+      class RenderTemplate < RenderNormalizer
+        register "render_template.action_view"
+
+        CAT = "view.render.template".freeze
+
+        # @param trace [Skylight::Messages::Trace::Builder] ignored, only present to match API
+        # @param name [String] ignored, only present to match API
+        # @param payload (see RenderNormalizer#normalize_render)
+        # @option payload (see RenderNormalizer#normalize_render)
+        # @return (see RenderNormalizer#normalize_render)
+        def normalize(trace, name, payload)
+          normalize_render(CAT, payload)
+        end
+      end
+    end
+  end
+end

data/lib/skylight/core/normalizers/active_job/enqueue_at.rb

@@ -0,0 +1,21 @@
+module Skylight::Core
+  module Normalizers
+    module ActiveJob
+      class EnqueueAt < Normalizer
+        register "enqueue_at.active_job"
+
+        CAT = "other.job.enqueue_at".freeze
+
+        def normalize(_trace, _name, payload)
+					title = "Enqueue #{payload[:job].class}"
+
+					adapter_class_name = payload[:adapter].class.name
+					adapter_name = adapter_class_name.match(/^ActiveJob::QueueAdapters::(\w+)Adapter$/)[1].underscore
+					desc = "{ adapter: '#{adapter_name}', queue: '#{payload[:job].queue_name}' }"
+
+					[ CAT, title, desc ]
+        end
+      end
+    end
+  end
+end

data/lib/skylight/core/normalizers/active_model_serializers/render.rb

@@ -0,0 +1,26 @@
+module Skylight::Core
+  module Normalizers
+    module ActiveModelSerializers
+      class Render < Normalizer
+        register "render.active_model_serializers"
+
+        CAT = "view.render.active_model_serializers".freeze
+
+        def normalize(trace, name, payload)
+          serializer_class = payload[:serializer]
+
+          title = serializer_class.name.sub(/^ActiveModel::(Serializer::)?/, '')
+
+          if adapter_instance = payload[:adapter]
+            adapter_name = adapter_instance.class.name
+                              .sub(/^ActiveModel::Serializer::Adapter::/, '')
+                              .sub(/^ActiveModelSerializers::Adapter::/, '')
+            desc = "Adapter: #{adapter_name}"
+          end
+
+          [ CAT, title, desc ]
+        end
+      end
+    end
+  end
+end