skylight 4.3.0 → 5.0.0.beta4

data/lib/skylight/sidekiq.rb

@@ -0,0 +1,48 @@
+module Skylight
+  module Sidekiq
+    def self.add_middleware
+      unless defined?(::Sidekiq)
+        Skylight.warn "Skylight for Sidekiq is active, but Sidekiq is not defined."
+        return
+      end
+
+      ::Sidekiq.configure_server do |sidekiq_config|
+        Skylight.debug "Adding Sidekiq Middleware"
+
+        sidekiq_config.server_middleware do |chain|
+          # Put it at the front
+          chain.prepend ServerMiddleware
+        end
+      end
+    end
+
+    class ServerMiddleware
+      include Util::Logging
+
+      def call(worker, job, queue)
+        t { "Sidekiq middleware beginning trace" }
+        title = job["wrapped"] || job["class"]
+
+        # TODO: Using hints here would be ideal but requires further refactoring
+        meta =
+          if (source_location = worker.method(:perform).source_location)
+            { source_file: source_location[0], source_line: source_location[1] }
+          end
+
+        Skylight.trace(title, "app.sidekiq.worker", title, meta: meta, segment: queue, component: :worker) do |trace|
+          yield
+        rescue Exception # includes Sidekiq::Shutdown
+          trace.segment = "error" if trace
+          raise
+        end
+      end
+    end
+
+    ActiveSupport::Notifications.subscribe("started_instrumenter.skylight") \
+        do |_name, _started, _finished, _unique_id, payload|
+      if payload[:instrumenter].config.enable_sidekiq?
+        add_middleware
+      end
+    end
+  end
+end

data/lib/skylight/subscriber.rb

@@ -0,0 +1,110 @@
+module Skylight
+  # @api private
+  class Subscriber
+    include Util::Logging
+
+    attr_reader :config, :normalizers
+
+    def initialize(config, instrumenter)
+      @config       = config
+      @normalizers  = Normalizers.build(config)
+      @instrumenter = instrumenter
+      @subscribers  = []
+    end
+
+    def register!
+      unregister!
+      @normalizers.keys.each do |key| # rubocop:disable Style/HashEachMethods
+        @subscribers << ActiveSupport::Notifications.subscribe(key, self)
+      end
+    end
+
+    def unregister!
+      ActiveSupport::Notifications.unsubscribe @subscribers.shift until @subscribers.empty?
+    end
+
+    #
+    #
+    # ===== ActiveSupport::Notifications API
+    #
+    #
+
+    class Notification
+      attr_reader :name, :span
+
+      def initialize(name, span)
+        @name = name
+        @span = span
+      end
+    end
+
+    def start(name, _id, payload)
+      return if @instrumenter.disabled?
+      return unless (trace = @instrumenter.current_trace)
+
+      _start(trace, name, payload)
+    end
+
+    def finish(name, _id, payload)
+      return if @instrumenter.disabled?
+      return unless (trace = @instrumenter.current_trace)
+
+      while (curr = trace.notifications.pop)
+        next unless curr.name == name
+
+        meta = {}
+        meta[:exception] = payload[:exception] if payload[:exception]
+        meta[:exception_object] = payload[:exception_object] if payload[:exception_object]
+        trace.done(curr.span, meta) if curr.span
+        normalize_after(trace, curr.span, name, payload)
+        return
+      end
+    rescue Exception => e
+      error "Subscriber#finish error; msg=%s", e.message
+      debug "trace=%s", trace.inspect
+      debug "in:  name=%s", name.inspect
+      debug "in:  payload=%s", payload.inspect
+      t { e.backtrace.join("\n") }
+      nil
+    end
+
+    def publish(name, *args)
+      # Ignored for now because nothing in rails uses it
+    end
+
+    private
+
+      def normalize(*args)
+        @normalizers.normalize(*args)
+      end
+
+      def normalize_after(*args)
+        @normalizers.normalize_after(*args)
+      end
+
+      def _start(trace, name, payload)
+        result = normalize(trace, name, payload)
+
+        unless result == :skip
+          case result.size
+          when 3, 4
+            cat, title, desc, meta = result
+          else
+            raise "Invalid normalizer result: #{result.inspect}"
+          end
+
+          span = trace.instrument(cat, title, desc, meta)
+        end
+      rescue Exception => e
+        error "Subscriber#start error; msg=%s", e.message
+        debug "trace=%s", trace.inspect
+        debug "in:  name=%s", name.inspect
+        debug "in:  payload=%s", payload.inspect
+        debug "out: cat=%s, title=%s, desc=%s", cat.inspect, name.inspect, desc.inspect
+        t { e.backtrace.join("\n") }
+        nil
+      ensure
+        trace.notifications << Notification.new(name, span)
+      end
+  end
+end

data/lib/skylight/test.rb

@@ -0,0 +1,146 @@
+module Skylight
+  module Test
+    module Mocking
+      def mock!(config_opts = {}, &callback)
+        config_opts[:mock_submission] ||= callback || proc {}
+
+        config_class = Class.new(Config) do
+          def validate_with_server
+            true
+          end
+        end
+
+        config = config_class.load(config_opts)
+        config[:authentication] ||= "zomg"
+
+        class_eval do
+          unless const_defined?(:OriginalInstrumenter)
+            const_set :OriginalInstrumenter, Instrumenter
+            remove_const :Instrumenter
+            const_set(:Instrumenter, Class.new(OriginalInstrumenter) do
+              def self.name
+                "Mocked Instrumenter"
+              end
+
+              def self.native_new(*)
+                allocate
+              end
+
+              def native_start
+                true
+              end
+
+              def native_submit_trace(trace)
+                config[:mock_submission].call(trace)
+              end
+
+              def native_stop; end
+            end)
+
+            const_set :OriginalTrace, Trace
+            remove_const :Trace
+            const_set(:Trace, Class.new(OriginalTrace) do
+              def self.native_new(start, _uuid, endpoint, meta)
+                inst = allocate
+                inst.instance_variable_set(:@start, start)
+                inst.instance_variable_set(:@endpoint, endpoint)
+                inst.instance_variable_set(:@starting_endpoint, endpoint)
+                inst.instance_variable_set(:@meta, meta)
+                inst
+              end
+
+              attr_reader :endpoint, :starting_endpoint, :meta
+
+              def mock_spans
+                @mock_spans ||= []
+              end
+
+              def filter_spans
+                if block_given?
+                  mock_spans.select { |span| yield span }
+                else
+                  mock_spans.reject { |span| span[:cat] == "noise.gc" }
+                end
+              end
+
+              def native_get_uuid
+                @uuid
+              end
+
+              def uuid=(value)
+                @uuid = value
+              end
+
+              def native_get_started_at
+                @start
+              end
+
+              def native_set_endpoint(endpoint)
+                @endpoint = endpoint
+              end
+
+              def native_set_component(component)
+                @component = component
+              end
+
+              def native_start_span(time, cat)
+                span = {
+                  start: time,
+                  cat:   cat
+                }
+                mock_spans << span
+                # Return integer like the native method does
+                mock_spans.index(span)
+              end
+
+              def native_span_set_title(span, title)
+                mock_spans[span][:title] = title
+              end
+
+              def native_span_set_description(span, desc)
+                mock_spans[span][:desc] = desc
+              end
+
+              def native_span_set_meta(span, meta)
+                mock_spans[span][:meta] = meta
+              end
+
+              def native_span_started(span); end
+
+              def native_span_set_exception(span, exception_object, exception)
+                mock_spans[span][:exception_object] = exception_object
+                mock_spans[span][:exception] = exception
+              end
+
+              def native_stop_span(span, time)
+                span = mock_spans[span]
+                span[:duration] = time - span[:start]
+                nil
+              end
+
+              def native_use_pruning
+                @using_native_pruning = true
+              end
+            end)
+          end
+        end
+
+        start!(config)
+      end
+
+      def unmock!
+        if const_defined?(:OriginalInstrumenter)
+          class_eval do
+            remove_const :Instrumenter
+            const_set :Instrumenter, OriginalInstrumenter
+            remove_const :OriginalInstrumenter
+
+            remove_const :Trace
+            const_set :Trace, OriginalTrace
+            remove_const :OriginalTrace
+          end
+        end
+      end
+    end
+  end
+end

data/lib/skylight/trace.rb

@@ -1,9 +1,55 @@
+require "securerandom"
+require "skylight/util/logging"
+
 module Skylight
-  class Trace < Core::Trace
-    attr_reader :component
+  class Trace
+    GC_CAT = "noise.gc".freeze
+    SYNTHETIC = "<synthetic>".freeze
+
+    META_KEYS = %i[mute_children database].freeze
+
+    include Util::Logging
+
+    attr_reader :instrumenter, :endpoint, :segment, :notifications, :meta, :component
+
+    def self.new(instrumenter, endpoint, start, cat, title = nil, desc = nil, meta: nil, segment: nil, component: nil)
+      uuid = SecureRandom.uuid
+      inst = native_new(normalize_time(start), uuid, endpoint, meta)
+      inst.uuid = uuid
+      inst.send(:initialize, instrumenter, cat, title, desc, meta, component: component)
+      inst.endpoint = endpoint
+      inst.segment = segment
+      inst
+    end
+
+    # TODO: Move this into native
+    def self.normalize_time(time)
+      # At least one customer has extensions that cause integer division to produce rationals.
+      # Since the native code expects an integer, we force it again.
+      (time.to_i / 100_000).to_i
+    end
+
+    def initialize(instrumenter, cat, title, desc, meta, component: nil)
+      raise ArgumentError, "instrumenter is required" unless instrumenter
+
+      @instrumenter = instrumenter
+      @submitted = false
+      @broken = false
+
+      @notifications = []
+
+      @spans = []
+
+      preprocess_meta(meta) if meta
+
+      # create the root node
+      @root = start(native_get_started_at, cat, title, desc, meta, normalize: false)
+
+      # Also store meta for later access
+      @meta = meta
+
+      @gc = config.gc.track unless ENV.key?("SKYLIGHT_DISABLE_GC_TRACKING")
 
-    def initialize(*, component: nil)
-      super
       self.component = component if component
       @too_many_spans = false
       native_use_pruning if use_pruning?
@@ -26,29 +72,248 @@ module Skylight
       !!@too_many_spans
     end
 
-    def maybe_broken(error)
-      if error.is_a?(Skylight::MaximumTraceSpansError) && config.get(:report_max_spans_exceeded)
+    def log_context
+      @log_context ||= { trace: uuid }
+    end
+
+    def endpoint=(value)
+      if muted?
+        maybe_warn(:endpoint_set_muted, "tried to set endpoint name while muted")
+        return
+      end
+      @endpoint = value
+      native_set_endpoint(value)
+    end
+
+    def segment=(value)
+      if muted?
+        maybe_warn(:segment_set_muted, "tried to set segment name while muted")
+        return
+      end
+      @segment = value
+    end
+
+    attr_accessor :compound_response_error_status
+
+    def config
+      @instrumenter.config
+    end
+
+    def muted?
+      !!@child_instrumentation_muted_by || @instrumenter.muted?
+    end
+
+    def broken?
+      !!@broken
+    end
+
+    def maybe_broken(err)
+      if err.is_a?(Skylight::MaximumTraceSpansError) && config.get(:report_max_spans_exceeded)
         too_many_spans!
       else
-        super
+        error "failed to instrument span; msg=%s; endpoint=%s", err.message, endpoint
+        broken!
       end
     end
 
+    def instrument(cat, title = nil, desc = nil, meta = nil)
+      return if muted?
+      return if broken?
+
+      t { "instrument: #{cat}, #{title}" }
+
+      title.freeze if title.is_a?(String)
+      desc.freeze  if desc.is_a?(String)
+
+      now = Skylight::Util::Clock.nanos
+
+      preprocess_meta(meta) if meta
+
+      start(now - gc_time, cat, title, desc, meta)
+    rescue => e
+      maybe_broken(e)
+      nil
+    end
+
+    def done(span, meta = nil)
+      # `span` will be `nil` if we failed to start instrumenting, such as in
+      # the case of too many spans in a request.
+      return unless span
+      return if broken?
+
+      if meta&.[](:defer)
+        deferred_spans[span] ||= (Skylight::Util::Clock.nanos - gc_time)
+        return
+      end
+
+      if meta && (meta[:exception_object] || meta[:exception])
+        native_span_set_exception(span, meta[:exception_object], meta[:exception])
+      end
+
+      stop(span, Skylight::Util::Clock.nanos - gc_time)
+    rescue => e
+      error "failed to close span; msg=%s; endpoint=%s", e.message, endpoint
+      log_trace "Original Backtrace:\n#{e.backtrace.join("\n")}"
+      broken!
+      nil
+    end
+
+    def inspect
+      to_s
+    end
+
+    def release
+      t { "release; is_current=#{@instrumenter.current_trace == self}" }
+      return unless @instrumenter.current_trace == self
+
+      @instrumenter.current_trace = nil
+    end
+
+    def broken!
+      debug "trace is broken"
+      @broken = true
+    end
+
     def traced
       if too_many_spans?
         error("[E%04d] The request exceeded the maximum number of spans allowed. It will still " \
               "be tracked but with reduced information. endpoint=%s", Skylight::MaximumTraceSpansError.code, endpoint)
       end
 
-      super
+      gc = gc_time
+      now = Skylight::Util::Clock.nanos
+      track_gc(gc, now)
+      stop(@root, now)
+    end
+
+    def submit
+      t { "submitting trace" }
+
+      # This must always be called to clean up properly
+      release
+
+      if broken?
+        t { "broken, not submitting" }
+        return
+      end
+
+      if @submitted
+        t { "already submitted" }
+        return
+      end
+
+      @submitted = true
+
+      traced
+
+      @instrumenter.process(self)
+    rescue Exception => e
+      error e.message
+      t { e.backtrace.join("\n") }
     end
 
     private
 
-      def track_gc(*)
+      def track_gc(time, now)
         # This attempts to log another span which will fail if we have too many
         return if too_many_spans?
-        super
+
+        if time > 0
+          t { fmt "tracking GC time; duration=%d", time }
+          meta = { source_location: SYNTHETIC }
+          stop(start(now - time, GC_CAT, nil, nil, meta), now)
+        end
+      end
+
+      def start(time, cat, title, desc, meta, opts = {})
+        time = self.class.normalize_time(time) unless opts[:normalize] == false
+
+        mute_children = meta&.delete(:mute_children)
+
+        sp = native_start_span(time, cat.to_s)
+        native_span_set_title(sp, title.to_s) if title
+        native_span_set_description(sp, desc.to_s) if desc
+        native_span_set_meta(sp, meta) if meta
+        native_span_started(sp)
+
+        @spans << sp
+        t { "started span: #{sp} - #{cat}, #{title}" }
+
+        if mute_children
+          t { "muting child instrumentation for span=#{sp}" }
+          mute_child_instrumentation(sp)
+        end
+
+        sp
+      end
+
+      def mute_child_instrumentation(span)
+        @child_instrumentation_muted_by = span
+      end
+
+      # Middleware spans that were interrupted by a throw/catch should be cached here.
+      # keys: span ids
+      # values: nsec timestamp at which the span was cached here.
+      def deferred_spans
+        @deferred_spans ||= {}
+      end
+
+      def stop(span, time)
+        t { "stopping span: #{span}" }
+
+        # If `stop` is called for a span that is not the last item in the stack,
+        # check to see if the last item has been marked as deferred. If so, close
+        # that span first, then try to close the original.
+        while deferred_spans[expected = @spans.pop]
+          normalized_stop(expected, deferred_spans.delete(expected))
+        end
+
+        handle_unexpected_stop(expected, span) unless span == expected
+
+        normalized_stop(span, time)
+        nil
+      end
+
+      def normalized_stop(span, time)
+        time = self.class.normalize_time(time)
+        native_stop_span(span, time)
+
+        if @child_instrumentation_muted_by == span
+          @child_instrumentation_muted_by = nil # restart instrumenting
+        end
+      end
+
+      # Originally extracted from `stop`.
+      # If we attempt to close spans out of order, and it appears to be a middleware issue,
+      # disable the middleware probe and mark trace as broken.
+      def handle_unexpected_stop(expected, span)
+        message = "[E0001] Spans were closed out of order. Expected to see '#{native_span_get_title(expected)}', " \
+                    "but got '#{native_span_get_title(span)}' instead."
+
+        if native_span_get_category(span) == "rack.middleware" && Skylight::Probes.installed.key?(:middleware)
+          if Skylight::Probes::Middleware::Probe.disabled?
+            message << "\nWe disabled the Middleware probe but unfortunately, this didn't solve the issue."
+          else
+            Skylight::Probes::Middleware::Probe.disable!
+            message << "\n#{native_span_get_title(span)} may be a Middleware that doesn't fully conform " \
+                        "to the Rack SPEC. We've disabled the Middleware probe to see if that resolves the issue."
+          end
+        end
+
+        message << "\nThis request will not be tracked. Please contact support@skylight.io for more information."
+
+        error message
+
+        t { "expected=#{expected}, actual=#{span}" }
+
+        broken!
+      end
+
+      def gc_time
+        return 0 unless @gc
+
+        @gc.update
+        @gc.time
       end
 
       def use_pruning?
@@ -61,9 +326,41 @@ module Skylight
 
       def component=(component)
         resolve_component(component).tap do |c|
+          # Would it be better for the component getter to get from native?
           @component = c
           native_set_component(c)
         end
       end
+
+      def preprocess_meta(meta)
+        validate_meta(meta)
+        instrumenter.extensions.trace_preprocess_meta(meta)
+      end
+
+      def validate_meta(meta)
+        unknown_keys = meta.keys - allowed_meta_keys
+        if unknown_keys.any?
+          unknown_keys.each do |key|
+            maybe_warn("unknown_meta:#{key}", "Unknown meta key will be ignored; key=#{key.inspect}")
+            meta.delete(key)
+          end
+        end
+      end
+
+      def allowed_meta_keys
+        META_KEYS | instrumenter.extensions.allowed_meta_keys
+      end
+
+      def maybe_warn(context, msg)
+        return if warnings_silenced?(context)
+
+        instrumenter.silence_warnings(context)
+
+        warn(msg)
+      end
+
+      def warnings_silenced?(context)
+        instrumenter.warnings_silenced?(context)
+      end
   end
 end