skylight 4.2.3 → 5.3.0

data/lib/skylight/deprecation.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require "active_support/deprecation"
+
+module Skylight
+  SKYLIGHT_GEM_ROOT = "#{File.expand_path("../..", __dir__)}/"
+
+  class Deprecation < ActiveSupport::Deprecation
+    private
+
+    def ignored_callstack(path)
+      path.start_with?(SKYLIGHT_GEM_ROOT)
+    end
+  end
+
+  DEPRECATOR = Deprecation.new("6.0", "skylight")
+end

data/lib/skylight/errors.rb

@@ -1,17 +1,21 @@
+require "json"
+
 module Skylight
+  # @api private
+  class ConfigError < RuntimeError
+  end
+
   class NativeError < StandardError
     @classes = {}
 
     def self.register(code, name, message)
-      if @classes.key?(code)
-        raise "Duplicate error class code: #{code}; name=#{name}"
-      end
+      raise "Duplicate error class code: #{code}; name=#{name}" if @classes.key?(code)
 
       Skylight.module_eval <<-RUBY, __FILE__, __LINE__ + 1
-        class #{name}Error < NativeError
-          def self.code; #{code}; end
-          def self.message; #{message.to_json}; end
-        end
+        class #{name}Error < NativeError            # class SqlLexError < NativeError
+          def self.code; #{code}; end               #   def self.code; 4; end
+          def self.message; #{message.to_json}; end #   def self.message; "Failed to lex SQL query."; end
+        end                                         # end
       RUBY
 
       klass = Skylight.const_get("#{name}Error")
@@ -29,13 +33,17 @@ module Skylight
       9999
     end
 
+    def self.formatted_code
+      format("%<code>04d", code: code)
+    end
+
     def self.message
       "Encountered an unknown internal error"
     end
 
     def initialize(method_name)
       @method_name = method_name
-      super(format("[E%<code>04d] %<message>s [%<meth>s]", code: code, message: message, meth: method_name))
+      super(format("[E%<code>04d] %<message>s [%<meth>s]", code: code, message: self.class.message, meth: method_name))
     end
 
     def code
@@ -43,9 +51,12 @@ module Skylight
     end
 
     def formatted_code
-      format("%04d", code)
+      self.class.formatted_code
     end
 
+    # E0002
+    # Too many unique descriptions - daemon only
+
     # E0003
     register(3, "MaximumTraceSpans", "Exceeded maximum number of spans in a trace.")
 
@@ -54,5 +65,11 @@ module Skylight
 
     # E0005
     register(5, "InstrumenterUnrecoverable", "Instrumenter is not running.")
+
+    # E0006
+    register(6, "InvalidUtf8", "Invalid UTF-8")
+
+    # E0007
+    register(7, "GrpcConnect", "Failed to connect to gRPC server.")
   end
 end

data/lib/skylight/extensions/source_location.rb

@@ -0,0 +1,291 @@
+# frozen_string_literal: true
+
+require "skylight/util/lru_cache"
+require "active_support/inflector"
+
+module Skylight
+  module Extensions
+    class SourceLocation < Extension
+      attr_reader :config
+
+      include Util::Logging
+
+      META_KEYS = %i[source_location source_file source_line].freeze
+      MAX_CALLER_DEPTH = 75
+
+      def initialize(*)
+        super
+        cache_size = (config[:source_location_cache_size] || 1000).to_i
+        @caller_cache = Util::LruCache.new(cache_size)
+        @instance_method_source_location_cache = Util::LruCache.new(cache_size)
+        gem_require_trie # memoize this at startup
+      end
+
+      def process_trace_meta(meta)
+        unless meta[:source_location] || meta[:source_file]
+          warn "Ignoring source_line without source_file" if meta[:source_line]
+          if (location = find_caller)
+            meta[:source_file] = location.absolute_path
+            meta[:source_line] = location.lineno
+          end
+        end
+      end
+
+      def process_instrument_options(opts, meta)
+        source_location = opts[:source_location] || opts[:meta]&.[](:source_location)
+        source_file = opts[:source_file] || opts[:meta]&.[](:source_file)
+        source_line = opts[:source_line] || opts[:meta]&.[](:source_line)
+        source_name_hint, const_name, method_name =
+          opts[:source_location_hint] || opts[:meta]&.[](:source_location_hint)
+        instrument_location = opts[:sk_instrument_location]
+
+        if source_location
+          meta[:source_location] = source_location
+        elsif source_name_hint
+          source_location = dispatch_hinted_source_location(source_name_hint, const_name, method_name)
+          meta[:source_file], meta[:source_line] = source_location
+          meta.delete(:source_location_hint) if source_location
+        elsif source_file
+          meta[:source_file] = source_file
+          meta[:source_line] = source_line
+        elsif instrument_location && project_path?(instrument_location.absolute_path)
+          meta[:source_file] = instrument_location.absolute_path
+          meta[:source_line] = instrument_location.lineno
+        else
+          warn "Ignoring source_line without source_file" if source_line
+          if (location = find_caller(cache_key: opts.hash))
+            meta[:source_file] = location.absolute_path
+            meta[:source_line] = location.lineno
+          end
+        end
+
+        meta
+      end
+
+      def process_normalizer_meta(payload, meta, **opts)
+        if opts[:source_location] && (opts[:source_file] || opts[:source_line])
+          warn "Found both source_location and source_file or source_line in normalizer\n" \
+                 "  location=#{opts[:source_location]}; file=#{opts[:source_file]}; line=#{opts[:source_line]}"
+        end
+
+        sl =
+          if (source_name, constant_name, method_name = opts[:source_location_hint])
+            dispatch_hinted_source_location(source_name, constant_name, method_name)
+          elsif opts[:source_file]
+            [opts[:source_file], opts[:source_line]]
+          end
+
+        sl ||= source_location(payload, meta, cache_key: opts[:cache_key])
+
+        if sl
+          trace("normalizer source_location=#{sl}")
+          meta[:source_file], meta[:source_line] = sl
+        end
+
+        meta
+      end
+
+      def trace_preprocess_meta(meta)
+        source_line = meta.delete(:source_line)
+        source_file = meta.delete(:source_file)
+
+        if meta[:source_location]
+          if source_file || source_line
+            warn "Found both source_location and source_file or source_line, using source_location\n" \
+                   "  location=#{meta[:source_location]}; file=#{source_file}; line=#{source_line}"
+          end
+
+          unless meta[:source_location].is_a?(String)
+            warn "Found non-string value for source_location; skipping"
+            meta.delete(:source_location)
+          end
+        elsif source_file
+          meta[:source_location] = sanitize_source_location(source_file, source_line)
+        elsif source_line
+          warn "Ignoring source_line without source_file; source_line=#{source_line}"
+        end
+
+        trace("source_location=#{meta[:source_location]}") if meta[:source_location]
+      end
+
+      def allowed_meta_keys
+        META_KEYS
+      end
+
+      protected
+
+      def dispatch_hinted_source_location(source_name, const_name, method_name)
+        return unless const_name && method_name
+
+        instance_method_source_location(const_name, method_name, source_name: source_name)
+      end
+
+      # from normalizers.rb
+      # Returns an array of file and line
+      def source_location(payload, meta, cache_key: nil)
+        # FIXME: what should precedence be?
+        if meta.is_a?(Hash) && meta[:source_location]
+          meta.delete(:source_location)
+        elsif payload.is_a?(Hash) && payload[:sk_source_location]
+          payload[:sk_source_location]
+        elsif (location = find_caller(cache_key: cache_key))
+          [location.absolute_path, location.lineno]
+        end
+      end
+
+      def find_caller(cache_key: nil)
+        # starting at 4 to skip Skylight extension processing logic
+        locations = ::Kernel.caller_locations(4..MAX_CALLER_DEPTH)
+
+        if cache_key
+          localized_cache_key = [cache_key, locations.map(&:lineno)].hash
+          @caller_cache.fetch(localized_cache_key) { find_caller_inner(locations) }
+        else
+          find_caller_inner(locations)
+        end
+      end
+
+      def project_path?(path)
+        return false unless path
+
+        # Must be in the project root
+        return false unless path.start_with?(config.root.to_s)
+
+        # Must not be Bundler's vendor location
+        return false if defined?(Bundler) && path.start_with?(Bundler.bundle_path.to_s)
+
+        # Must not be Ruby files
+        return false if path.include?("/ruby-#{RUBY_VERSION}/lib/ruby/")
+
+        # So it must be a project file
+        true
+      end
+
+      def instance_method_source_location(constant_name, method_name, source_name: :instance_method)
+        @instance_method_source_location_cache.fetch([constant_name, method_name, source_name]) do
+          if (constant = ::ActiveSupport::Inflector.safe_constantize(constant_name))
+            if constant.instance_methods.include?(:"before_instrument_#{method_name}")
+              method_name = :"before_instrument_#{method_name}"
+            end
+            begin
+              unbound_method =
+                case source_name
+                when :instance_method
+                  find_instance_method(constant, method_name)
+                when :own_instance_method
+                  find_own_instance_method(constant, method_name)
+                when :instance_method_super
+                  find_instance_method_super(constant, method_name)
+                when :class_method
+                  find_class_method(constant, method_name)
+                end
+
+              unbound_method&.source_location
+            rescue NameError
+              nil
+            end
+          end
+        end
+      end
+
+      def sanitize_source_location(path, line)
+        # Do this first since gems may be vendored in the app repo. However, it might be slower.
+        # Should we cache matches?
+        if (gem_name = find_source_gem(path))
+          path = gem_name
+          line = nil
+        elsif project_path?(path)
+          # Get relative path to root
+          path = Pathname.new(path).relative_path_from(config.root).to_s
+        else
+          return
+        end
+
+        line ? "#{path}:#{line}" : path
+      end
+
+      private
+
+      def gem_require_trie
+        @gem_require_trie ||=
+          begin
+            trie = {}
+
+            Gem.loaded_specs.each do |name, spec|
+              next if config.source_location_ignored_gems&.include?(name)
+
+              spec.full_require_paths.each do |path|
+                t1 = trie
+
+                path
+                  .split(File::SEPARATOR)
+                  .each do |segment|
+                    t1[segment] ||= {}
+                    t1 = t1[segment]
+                  end
+
+                t1[:name] = name
+              end
+            end
+
+            trie
+          end
+      end
+
+      def find_source_gem(path)
+        return nil unless path
+
+        trie = gem_require_trie
+
+        path
+          .split(File::SEPARATOR)
+          .each do |segment|
+            trie = trie[segment]
+            break unless trie
+            return trie[:name] if trie[:name]
+          end
+
+        nil
+      end
+
+      def find_caller_inner(locations)
+        # Start at file before this one
+        # NOTE: We could start farther back now to avoid more Skylight files
+        locations.find do |l|
+          absolute_path = l.absolute_path
+          find_source_gem(absolute_path) || project_path?(absolute_path)
+        end
+      end
+
+      # walks up the inheritance tree until it finds the last method
+      # without a super_method definition.
+      def find_instance_method_super(constant, method_name)
+        return unless (unbound_method = find_instance_method(constant, method_name))
+
+        unbound_method = unbound_method.super_method while unbound_method.super_method
+
+        unbound_method
+      end
+
+      # walks up the inheritance tree until it finds the instance method
+      # belonging to the constant given (skip prepended modules)
+      def find_own_instance_method(constant, method_name)
+        return unless (unbound_method = find_instance_method(constant, method_name))
+
+        while unbound_method.owner != constant && unbound_method.super_method
+          unbound_method = unbound_method.super_method
+        end
+
+        unbound_method if unbound_method.owner == constant
+      end
+
+      def find_instance_method(constant, method_name)
+        constant.instance_method(method_name)
+      end
+
+      def find_class_method(constant, method_name)
+        constant.method(method_name)
+      end
+    end
+  end
+end

data/lib/skylight/extensions.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+require "active_support/inflector"
+
+module Skylight
+  module Extensions
+    class Collection
+      def initialize(config, extensions = [])
+        @config = config
+        @extensions = extensions
+      end
+
+      def enable!(ext_name)
+        return if enabled?(ext_name)
+
+        find_by_name(ext_name) do |ext_class|
+          extensions << ext_class.new(config)
+          rememoize!
+        end
+      end
+
+      def disable!(ext_name)
+        find_by_name(ext_name) do |ext_class|
+          extensions.reject! { |x| x.is_a?(ext_class) }
+          rememoize!
+        end
+      end
+
+      def enabled?(ext_name)
+        return unless (ext_class = find_by_name(ext_name))
+
+        !!extensions.detect { |x| x.is_a?(ext_class) }
+      end
+
+      def process_trace_meta(meta)
+        extensions.each { |ext| ext.process_trace_meta(meta) }
+      end
+
+      # meta is a mutable hash that will be passed to the instrumenter.
+      # This method bridges Skylight.instrument and instrumenter.instrument.
+      def process_instrument_options(opts, meta)
+        extensions.each { |ext| ext.process_instrument_options(opts, meta) }
+      end
+
+      def process_normalizer_meta(payload, meta, **opts)
+        extensions.each { |ext| ext.process_normalizer_meta(payload, meta, **opts) }
+      end
+
+      def trace_preprocess_meta(meta)
+        extensions.each { |ext| ext.trace_preprocess_meta(meta) }
+      end
+
+      def allowed_meta_keys
+        @allowed_meta_keys ||= extensions.flat_map(&:allowed_meta_keys).uniq
+      end
+
+      private
+
+      attr_reader :extensions, :config
+
+      def find_by_name(ext_name)
+        begin
+          Skylight::Extensions.const_get(ActiveSupport::Inflector.classify(ext_name))
+        rescue NameError
+          return nil
+        end.tap { |const| yield const if block_given? }
+      end
+
+      def rememoize!
+        @allowed_meta_keys = nil
+        allowed_meta_keys
+      end
+    end
+
+    class Extension
+      def initialize(config)
+        @config = config
+      end
+
+      def process_trace_meta(_meta); end
+
+      def process_instrument_options(_opts, _meta); end
+
+      def process_normalizer_meta(_payload, _meta, **opts); end
+
+      def trace_preprocess_meta(_meta); end
+
+      def allowed_meta_keys
+        []
+      end
+    end
+  end
+end
+
+require "skylight/extensions/source_location"

data/lib/skylight/formatters/http.rb

@@ -0,0 +1,18 @@
+module Skylight
+  module Formatters
+    module HTTP
+      # Build instrumentation options for HTTP queries
+      #
+      # @param [String] method HTTP method, e.g. get, post
+      # @param [String] scheme HTTP scheme, e.g. http, https
+      # @param [String] host Request host, e.g. example.com
+      # @param [String, Integer] port Request port
+      # @param [String] path Request path
+      # @param [String] query Request query string
+      # @return [Hash] a hash containing `:category`, `:title`, and `:annotations`
+      def self.build_opts(method, _scheme, host, _port, _path, _query)
+        { category: "api.http.#{method.downcase}", title: "#{method.upcase} #{host}", internal: true }
+      end
+    end
+  end
+end

data/lib/skylight/gc.rb

@@ -0,0 +1,99 @@
+require "skylight/util/logging"
+
+module Skylight
+  # @api private
+  class GC
+    METHODS = %i[enable total_time].freeze
+    TH_KEY = :SK_GC_CURR_WINDOW
+    MAX_COUNT = 1000
+    MAX_TIME = 30_000_000
+
+    include Util::Logging
+
+    attr_reader :config
+
+    def initialize(config, profiler)
+      @listeners = []
+      @config = config
+      @lock = Mutex.new
+      @time = 0
+
+      if METHODS.all? { |m| profiler.respond_to?(m) }
+        @profiler = profiler
+        @time = @profiler.total_time
+      else
+        debug "disabling GC profiling"
+      end
+    end
+
+    def enable
+      @profiler&.enable
+    end
+
+    # Total time in microseconds for GC over entire process lifetime
+    def total_time
+      @profiler ? @profiler.total_time : nil
+    end
+
+    def track
+      if @profiler
+        win = Window.new(self)
+
+        @lock.synchronize do
+          __update
+          @listeners << win
+
+          # Cleanup any listeners that might have leaked
+          @listeners.shift until @listeners[0].time < MAX_TIME
+
+          @listeners.shift if @listeners.length > MAX_COUNT
+        end
+
+        win
+      else
+        Window.new(nil)
+      end
+    end
+
+    def release(win)
+      @lock.synchronize { @listeners.delete(win) }
+    end
+
+    def update
+      @lock.synchronize { __update }
+
+      nil
+    end
+
+    private
+
+    def __update
+      time = @profiler.total_time
+      diff = time - @time
+      @time = time
+
+      @listeners.each { |l| l.add(diff) } if diff > 0
+    end
+
+    class Window
+      attr_reader :time
+
+      def initialize(global)
+        @global = global
+        @time = 0
+      end
+
+      def update
+        @global&.update
+      end
+
+      def add(time)
+        @time += time
+      end
+
+      def release
+        @global&.release(self)
+      end
+    end
+  end
+end