skylight 4.3.2 → 5.0.1

data/lib/skylight/formatters/http.rb

@@ -0,0 +1,20 @@
+module Skylight
+  module Formatters
+    module HTTP
+      # Build instrumentation options for HTTP queries
+      #
+      # @param [String] method HTTP method, e.g. get, post
+      # @param [String] scheme HTTP scheme, e.g. http, https
+      # @param [String] host Request host, e.g. example.com
+      # @param [String, Integer] port Request port
+      # @param [String] path Request path
+      # @param [String] query Request query string
+      # @return [Hash] a hash containing `:category`, `:title`, and `:annotations`
+      def self.build_opts(method, _scheme, host, _port, _path, _query)
+        { category: "api.http.#{method.downcase}",
+          title:    "#{method.upcase} #{host}",
+          internal: true }
+      end
+    end
+  end
+end

data/lib/skylight/gc.rb

@@ -0,0 +1,109 @@
+require "skylight/util/logging"
+
+module Skylight
+  # @api private
+  class GC
+    METHODS   = %i[enable total_time].freeze
+    TH_KEY    = :SK_GC_CURR_WINDOW
+    MAX_COUNT = 1000
+    MAX_TIME  = 30_000_000
+
+    include Util::Logging
+
+    attr_reader :config
+
+    def initialize(config, profiler)
+      @listeners = []
+      @config    = config
+      @lock      = Mutex.new
+      @time      = 0
+
+      if METHODS.all? { |m| profiler.respond_to?(m) }
+        @profiler = profiler
+        @time = @profiler.total_time
+      else
+        debug "disabling GC profiling"
+      end
+    end
+
+    def enable
+      @profiler&.enable
+    end
+
+    # Total time in microseconds for GC over entire process lifetime
+    def total_time
+      @profiler ? @profiler.total_time : nil
+    end
+
+    def track
+      if @profiler
+        win = Window.new(self)
+
+        @lock.synchronize do
+          __update
+          @listeners << win
+
+          # Cleanup any listeners that might have leaked
+          @listeners.shift until @listeners[0].time < MAX_TIME
+
+          if @listeners.length > MAX_COUNT
+            @listeners.shift
+          end
+        end
+
+        win
+      else
+        Window.new(nil)
+      end
+    end
+
+    def release(win)
+      @lock.synchronize do
+        @listeners.delete(win)
+      end
+    end
+
+    def update
+      @lock.synchronize do
+        __update
+      end
+
+      nil
+    end
+
+    private
+
+      def __update
+        time  = @profiler.total_time
+        diff  = time - @time
+        @time = time
+
+        if diff > 0
+          @listeners.each do |l|
+            l.add(diff)
+          end
+        end
+      end
+
+      class Window
+        attr_reader :time
+
+        def initialize(global)
+          @global = global
+          @time   = 0
+        end
+
+        def update
+          @global&.update
+        end
+
+        def add(time)
+          @time += time
+        end
+
+        def release
+          @global&.release(self)
+        end
+      end
+  end
+end

data/lib/skylight/helpers.rb

@@ -14,7 +14,7 @@ module Skylight
 
         if (opts = @__sk_instrument_next_method)
           @__sk_instrument_next_method = nil
-          instrument_method(name, opts)
+          instrument_method(name, **opts)
         end
       end
 
@@ -24,7 +24,7 @@ module Skylight
 
         if (opts = @__sk_instrument_next_method)
           @__sk_instrument_next_method = nil
-          instrument_class_method(name, opts)
+          instrument_class_method(name, **opts)
         end
       end
 
@@ -77,14 +77,12 @@ module Skylight
       #         do_expensive_stuff
       #       end
       #     end
-      def instrument_method(*args)
-        opts = args.pop if args.last.is_a?(Hash)
-
+      def instrument_method(*args, **opts)
         if (name = args.pop)
           title = "#{self}##{name}"
-          __sk_instrument_method_on(self, name, title, opts || {})
+          __sk_instrument_method_on(self, name, title, **opts)
         else
-          @__sk_instrument_next_method = opts || {}
+          @__sk_instrument_next_method = opts
         end
       end
 
@@ -123,37 +121,82 @@ module Skylight
       #
       #       instrument_class_method :my_method, title: 'Expensive work'
       #     end
-      def instrument_class_method(name, opts = {})
+      def instrument_class_method(name, **opts)
+        # NOTE: If the class is defined anonymously and then assigned to a variable this code
+        #   will not be aware of the updated name.
         title = "#{self}.#{name}"
-        __sk_instrument_method_on(__sk_singleton_class, name, title, opts || {})
+        __sk_instrument_method_on(__sk_singleton_class, name, title, **opts)
       end
 
       private
 
-        def __sk_instrument_method_on(klass, name, title, opts)
+        HAS_ARGUMENT_FORWARDING = Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("2.7.0")
+
+        def __sk_instrument_method_on(klass, name, title, **opts)
           category = (opts[:category] || "app.method").to_s
           title    = (opts[:title] || title).to_s
           desc     = opts[:description].to_s if opts[:description]
 
-          klass.class_eval <<-RUBY, __FILE__, __LINE__ + 1
-            alias_method :"before_instrument_#{name}", :"#{name}"
+          # NOTE: The source location logic happens before we have have a config so we can'
+          # check if source locations are enabled. However, it only happens once so the potential impact
+          # should be minimal. This would more appropriately belong to Extensions::SourceLocation,
+          # but as that is a runtime concern, and this happens at compile time, there isn't currently
+          # a clean way to turn this on and off. The absence of the extension will cause the
+          # source_file and source_line to be removed from the trace span before it is submitted.
+          source_file, source_line = klass.instance_method(name).source_location
 
-            def #{name}(*args, &blk)
-              span = Skylight.instrument(
-                category:  :"#{category}",
-                title:       #{title.inspect},
-                description: #{desc.inspect})
+          # We should strongly prefer using the new argument-forwarding syntax (...) where available.
+          # In Ruby 2.7, the following are known to be syntax errors:
+          #
+          # - mixing positional arguments with argument forwarding (e.g., send(:method_name, ...))
+          # - calling a setter method with multiple arguments, unless dispatched via send or public_send.
+          #
+          # So it is possible, though not recommended, to define setter methods that take multiple arguments,
+          # keywords, and/or blocks. Unfortunately, this means that for setters, we still need to explicitly
+          # forward the different argument types.
+          is_setter_method = name.to_s.end_with?("=")
 
-              meta = {}
-              begin
-                send(:before_instrument_#{name}, *args, &blk)
-              rescue Exception => e
-                meta[:exception_object] = e
-                raise e
-              ensure
-                Skylight.done(span, meta) if span
-              end
+          arg_string =
+            if HAS_ARGUMENT_FORWARDING
+              is_setter_method ? "*args, **kwargs, &blk" : "..."
+            else
+              "*args, &blk"
             end
+
+          original_method_dispatch =
+            if is_setter_method
+              "self.send(:before_instrument_#{name}, #{arg_string})"
+            else
+              "before_instrument_#{name}(#{arg_string})"
+            end
+
+          klass.class_eval <<-RUBY, __FILE__, __LINE__ + 1
+            alias_method :"before_instrument_#{name}", :"#{name}"       # alias_method :"before_instrument_process", :"process"
+            def #{name}(#{arg_string})                                  # def process(*args, **kwargs, &blk)
+              span = Skylight.instrument(                               #   span = Skylight.instrument(
+                category:    :"#{category}",                            #     category:    :"app.method",
+                title:       #{title.inspect},                          #     title:       "process",
+                description: #{desc.inspect},                           #     description: "Process data",
+                source_file: #{source_file.inspect},                    #     source_file: "myapp/lib/processor.rb",
+                source_line: #{source_line.inspect})                    #     source_line: 123)
+                                                                        #
+              meta = {}                                                 #   meta = {}
+                                                                        #
+              begin                                                     #   begin
+                #{original_method_dispatch}                             #     self.before_instrument_process(...)
+              rescue Exception => e                                     #   rescue Exception => e
+                meta[:exception_object] = e                             #     meta[:exception_object] = e
+                raise e                                                 #     raise e
+              ensure                                                    #   ensure
+                Skylight.done(span, meta) if span                       #     Skylight.done(span, meta) if span
+              end                                                       #   end
+            end                                                         # end
+                                                                        #
+            if protected_method_defined?(:"before_instrument_#{name}")  # if protected_method_defined?(:"before_instrument_process")
+              protected :"#{name}"                                      #   protected :"process"
+            elsif private_method_defined?(:"before_instrument_#{name}") # elsif private_method_defined?(:"before_instrument_process")
+              private :"#{name}"                                        #   private :"process"
+            end                                                         # end
           RUBY
         end
 

data/lib/skylight/instrumenter.rb

@@ -1,10 +1,109 @@
+require "strscan"
+require "securerandom"
+require "skylight/util/logging"
+require "skylight/extensions"
+
 module Skylight
-  class Instrumenter < Core::Instrumenter
-    def self.trace_class
-      Trace
+  # @api private
+  class Instrumenter
+    KEY = :__skylight_current_trace
+
+    include Util::Logging
+
+    class TraceInfo
+      def initialize(key = KEY)
+        @key = key
+        @muted_key = "#{key}_muted"
+      end
+
+      def current
+        Thread.current[@key]
+      end
+
+      def current=(trace)
+        Thread.current[@key] = trace
+      end
+
+      # NOTE: This should only be set by the instrumenter, and only
+      # in the context of a `mute` block. Do not try to turn this
+      # flag on and off directly.
+      def muted=(val)
+        Thread.current[@muted_key] = val
+      end
+
+      def muted?
+        !!Thread.current[@muted_key]
+      end
+    end
+
+    attr_reader :uuid, :config, :gc, :extensions
+
+    def self.native_new(_uuid, _config_env)
+      raise "not implemented"
+    end
+
+    def self.new(config)
+      config.validate!
+
+      uuid = SecureRandom.uuid
+      inst = native_new(uuid, config.to_native_env)
+      inst.send(:initialize, uuid, config)
+      inst
+    end
+
+    def initialize(uuid, config)
+      @uuid = uuid
+      @gc = config.gc
+      @config = config
+      @subscriber = Skylight::Subscriber.new(config, self)
+
+      @trace_info = @config[:trace_info] || TraceInfo.new(KEY)
+      @mutex = Mutex.new
+      @extensions = Skylight::Extensions::Collection.new(@config)
+    end
+
+    def enable_extension!(name)
+      @mutex.synchronize { extensions.enable!(name) }
+    end
+
+    def disable_extension!(name)
+      @mutex.synchronize { extensions.disable!(name) }
+    end
+
+    def extension_enabled?(name)
+      extensions.enabled?(name)
+    end
+
+    def log_context
+      @log_context ||= { inst: uuid }
+    end
+
+    def native_start
+      raise "not implemented"
     end
 
-    def check_install!
+    def native_stop
+      raise "not implemented"
+    end
+
+    def native_track_desc(_endpoint, _description)
+      raise "not implemented"
+    end
+
+    def native_submit_trace(_trace)
+      raise "not implemented"
+    end
+
+    def current_trace
+      @trace_info.current
+    end
+
+    def current_trace=(trace)
+      t { "setting current_trace=#{trace ? trace.uuid : 'nil'}; thread=#{Thread.current.object_id}" }
+      @trace_info.current = trace
+    end
+
+    def validate_installation
       # Warn if there was an error installing Skylight.
 
       if defined?(Skylight.check_install_errors)
@@ -13,24 +112,236 @@ module Skylight
 
       if !Skylight.native? && defined?(Skylight.warn_skylight_native_missing)
         Skylight.warn_skylight_native_missing(config)
-        return
+        return false
       end
+
+      true
+    end
+
+    def muted=(val)
+      @trace_info.muted = val
+    end
+
+    def muted?
+      @trace_info.muted?
     end
 
-    def process_sql(sql)
-      Skylight.lex_sql(sql)
-    rescue SqlLexError => e
-      if config[:log_sql_parse_errors]
-        config.logger.error "[#{e.formatted_code}] Failed to extract binds from SQL query. " \
-                            "It's likely that this query uses more advanced syntax than we currently support. " \
-                            "sql=#{sql.inspect}"
+    def mute
+      old_muted = muted?
+      self.muted = true
+      yield if block_given?
+    ensure
+      self.muted = old_muted
+    end
+
+    def unmute
+      old_muted = muted?
+      self.muted = false
+      yield if block_given?
+    ensure
+      self.muted = old_muted
+    end
+
+    def silence_warnings(context)
+      @warnings_silenced || @mutex.synchronize do
+        @warnings_silenced ||= {}
+      end
+
+      @warnings_silenced[context] = true
+    end
+
+    def warnings_silenced?(context)
+      @warnings_silenced && @warnings_silenced[context]
+    end
+
+    alias disable mute
+    alias disabled? muted?
+
+    def start!
+      # We do this here since we can't report these issues via Gem install without stopping install entirely.
+      return unless validate_installation
+
+      t { "starting instrumenter" }
+
+      unless config.validate_with_server
+        log_error "invalid config"
+        return
+      end
+
+      t { "starting native instrumenter" }
+      unless native_start
+        warn "failed to start instrumenter"
+        return
       end
+
+      enable_extension!(:source_location) if @config.enable_source_locations?
+      config.gc.enable
+      @subscriber.register!
+
+      ActiveSupport::Notifications.instrument("started_instrumenter.skylight", instrumenter: self)
+
+      self
+    rescue Exception => e
+      log_error "failed to start instrumenter; msg=%s; config=%s", e.message, config.inspect
+      t { e.backtrace.join("\n") }
       nil
     end
 
-    def handle_instrumenter_error(trace, e)
-      poison! if e.is_a?(Skylight::InstrumenterUnrecoverableError)
-      super
+    def shutdown
+      @subscriber.unregister!
+      native_stop
+    end
+
+    def trace(endpoint, cat, title = nil, desc = nil, meta: nil, segment: nil, component: nil)
+      # If a trace is already in progress, continue with that one
+      if (trace = @trace_info.current)
+        return yield(trace) if block_given?
+
+        return trace
+      end
+
+      begin
+        meta ||= {}
+        extensions.process_trace_meta(meta)
+        trace = Trace.new(self, endpoint, Skylight::Util::Clock.nanos, cat, title, desc,
+                          meta: meta, segment: segment, component: component)
+      rescue Exception => e
+        log_error e.message
+        t { e.backtrace.join("\n") }
+        return
+      end
+
+      @trace_info.current = trace
+      return trace unless block_given?
+
+      begin
+        yield trace
+      ensure
+        @trace_info.current = nil
+        t { "instrumenter submitting trace; trace=#{trace.uuid}" }
+        trace.submit
+      end
+    end
+
+    def instrument(cat, title = nil, desc = nil, meta = nil)
+      raise ArgumentError, "cat is required" unless cat
+
+      if muted?
+        return yield if block_given?
+
+        return
+      end
+
+      unless (trace = @trace_info.current)
+        return yield if block_given?
+
+        return
+      end
+
+      cat = cat.to_s
+
+      unless Skylight::CATEGORY_REGEX.match?(cat)
+        warn "invalid skylight instrumentation category; trace=%s; value=%s", trace.uuid, cat
+        return yield if block_given?
+
+        return
+      end
+
+      cat = "other.#{cat}" unless Skylight::TIER_REGEX.match?(cat)
+
+      unless (sp = trace.instrument(cat, title, desc, meta))
+        return yield if block_given?
+
+        return
+      end
+
+      return sp unless block_given?
+
+      begin
+        yield sp
+      rescue Exception => e
+        meta ||= {}
+        meta[:exception] = [e.class.name, e.message]
+        meta[:exception_object] = e
+        raise e
+      ensure
+        trace.done(sp, meta)
+      end
+    end
+
+    def broken!
+      return unless (trace = @trace_info.current)
+
+      trace.broken!
+    end
+
+    def poison!
+      @poisoned = true
+    end
+
+    def poisoned?
+      @poisoned
+    end
+
+    def done(span, meta = nil)
+      return unless (trace = @trace_info.current)
+
+      trace.done(span, meta)
+    end
+
+    def process(trace)
+      t { fmt "processing trace=#{trace.uuid}" }
+
+      if ignore?(trace)
+        t { fmt "ignoring trace=#{trace.uuid}" }
+        return false
+      end
+
+      begin
+        finalize_endpoint_segment(trace)
+        native_submit_trace(trace)
+        true
+      rescue => e
+        handle_instrumenter_error(trace, e)
+      end
+    end
+
+    def handle_instrumenter_error(trace, err)
+      poison! if err.is_a?(Skylight::InstrumenterUnrecoverableError)
+
+      warn "failed to submit trace to worker; trace=%s, err=%s", trace.uuid, err
+      t { "BACKTRACE:\n#{err.backtrace.join("\n")}" }
+
+      false
+    end
+
+    def ignore?(trace)
+      config.ignored_endpoints.include?(trace.endpoint)
+    end
+
+    # Because GraphQL can return multiple results, each of which
+    # may have their own success/error states, we need to set the
+    # skylight segment as follows:
+    #
+    # - when all queries have errors: "error"
+    # - when some queries have errors: "<rendered format>+error"
+    # - when no queries have errors: "<rendered format>"
+    #
+    # <rendered format> will be determined by the Rails controller as usual.
+    # See Instrumenter#finalize_endpoint_segment for the actual segment/error assignment.
+    def finalize_endpoint_segment(trace)
+      return unless (segment = trace.segment)
+
+      segment = case trace.compound_response_error_status
+                when :all
+                  "error"
+                when :partial
+                  "#{segment}+error"
+                else
+                  segment
+                end
+
+      trace.endpoint += "<sk-segment>#{segment}</sk-segment>"
     end
   end
 end