skylight 4.2.3 → 5.3.0

data/lib/skylight/trace.rb

@@ -1,9 +1,55 @@
+require "securerandom"
+require "skylight/util/logging"
+
 module Skylight
-  class Trace < Core::Trace
-    attr_reader :component
+  class Trace
+    GC_CAT = "noise.gc".freeze
+    SYNTHETIC = "<synthetic>".freeze
+
+    META_KEYS = %i[mute_children database].freeze
+
+    include Util::Logging
+
+    attr_reader :instrumenter, :endpoint, :segment, :notifications, :meta, :component
+
+    def self.new(instrumenter, endpoint, start, cat, title = nil, desc = nil, meta: nil, segment: nil, component: nil)
+      uuid = SecureRandom.uuid
+      inst = native_new(normalize_time(start), uuid, endpoint, meta)
+      inst.uuid = uuid
+      inst.send(:initialize, instrumenter, cat, title, desc, meta, component: component)
+      inst.endpoint = endpoint
+      inst.segment = segment
+      inst
+    end
+
+    # TODO: Move this into native
+    def self.normalize_time(time)
+      # At least one customer has extensions that cause integer division to produce rationals.
+      # Since the native code expects an integer, we force it again.
+      (time.to_i / 100_000).to_i
+    end
+
+    def initialize(instrumenter, cat, title, desc, meta, component: nil)
+      raise ArgumentError, "instrumenter is required" unless instrumenter
+
+      @instrumenter = instrumenter
+      @submitted = false
+      @broken = false
+
+      @notifications = []
+
+      @spans = []
+
+      preprocess_meta(meta) if meta
+
+      # create the root node
+      @root = start(native_get_started_at, cat, title, desc, meta, normalize: false)
+
+      # Also store meta for later access
+      @meta = meta
+
+      @gc = config.gc.track unless ENV.key?("SKYLIGHT_DISABLE_GC_TRACKING")
 
-    def initialize(*, component: nil)
-      super
       self.component = component if component
       @too_many_spans = false
       native_use_pruning if use_pruning?
@@ -26,44 +72,301 @@ module Skylight
       !!@too_many_spans
     end
 
-    def maybe_broken(error)
-      if error.is_a?(Skylight::MaximumTraceSpansError) && config.get(:report_max_spans_exceeded)
+    def log_context
+      @log_context ||= { trace: uuid }
+    end
+
+    def endpoint=(value)
+      if muted?
+        maybe_warn(:endpoint_set_muted, "tried to set endpoint name while muted")
+        return
+      end
+      @endpoint = value
+      native_set_endpoint(value)
+    end
+
+    def segment=(value)
+      if muted?
+        maybe_warn(:segment_set_muted, "tried to set segment name while muted")
+        return
+      end
+      @segment = value
+    end
+
+    attr_accessor :compound_response_error_status
+
+    def config
+      @instrumenter.config
+    end
+
+    def muted?
+      !!@child_instrumentation_muted_by || @instrumenter.muted?
+    end
+
+    def broken?
+      !!@broken
+    end
+
+    def maybe_broken(err)
+      if err.is_a?(Skylight::MaximumTraceSpansError) && config.get(:report_max_spans_exceeded)
         too_many_spans!
       else
-        super
+        error "failed to instrument span; msg=%s; endpoint=%s", err.message, endpoint
+        broken!
+      end
+    end
+
+    def instrument(cat, title = nil, desc = nil, meta = nil)
+      return if muted?
+      return if broken?
+
+      t { "instrument: #{cat}, #{title}" }
+
+      title.freeze if title.is_a?(String)
+      desc.freeze if desc.is_a?(String)
+
+      now = Skylight::Util::Clock.nanos
+
+      preprocess_meta(meta) if meta
+
+      start(now - gc_time, cat, title, desc, meta)
+    rescue StandardError => e
+      maybe_broken(e)
+      nil
+    end
+
+    def done(span, meta = nil)
+      # `span` will be `nil` if we failed to start instrumenting, such as in
+      # the case of too many spans in a request.
+      return unless span
+      return if broken?
+
+      if meta&.[](:defer)
+        deferred_spans[span] ||= (Skylight::Util::Clock.nanos - gc_time)
+        return
       end
+
+      if meta && (meta[:exception_object] || meta[:exception])
+        native_span_set_exception(span, meta[:exception_object], meta[:exception])
+      end
+
+      stop(span, Skylight::Util::Clock.nanos - gc_time)
+    rescue StandardError => e
+      error "failed to close span; msg=%s; endpoint=%s", e.message, endpoint
+      log_trace "Original Backtrace:\n#{e.backtrace.join("\n")}"
+      broken!
+      nil
+    end
+
+    def inspect
+      to_s
+    end
+
+    def release
+      t { "release; is_current=#{@instrumenter.current_trace == self}" }
+      return unless @instrumenter.current_trace == self
+
+      @instrumenter.current_trace = nil
+    end
+
+    def broken!
+      debug "trace is broken"
+      @broken = true
     end
 
     def traced
       if too_many_spans?
-        error("[E%04d] The request exceeded the maximum number of spans allowed. It will still " \
-              "be tracked but with reduced information. endpoint=%s", Skylight::MaximumTraceSpansError.code, endpoint)
+        error(
+          "[E%04d] The request exceeded the maximum number of spans allowed. It will still " \
+            "be tracked but with reduced information. endpoint=%s",
+          Skylight::MaximumTraceSpansError.code,
+          endpoint
+        )
+      end
+
+      gc = gc_time
+      now = Skylight::Util::Clock.nanos
+      track_gc(gc, now)
+      stop(@root, now)
+    end
+
+    def submit
+      t { "submitting trace" }
+
+      # This must always be called to clean up properly
+      release
+
+      if broken?
+        t { "broken, not submitting" }
+        return
+      end
+
+      if @submitted
+        t { "already submitted" }
+        return
       end
 
-      super
+      @submitted = true
+
+      traced
+
+      @instrumenter.process(self)
+    rescue Exception => e
+      error e.message
+      t { e.backtrace.join("\n") }
     end
 
     private
 
-      def track_gc(*)
-        # This attempts to log another span which will fail if we have too many
-        return if too_many_spans?
-        super
+    def track_gc(time, now)
+      # This attempts to log another span which will fail if we have too many
+      return if too_many_spans?
+
+      if time > 0
+        t { fmt "tracking GC time; duration=%d", time }
+        meta = { source_location: SYNTHETIC }
+        stop(start(now - time, GC_CAT, nil, nil, meta), now)
+      end
+    end
+
+    def start(time, cat, title, desc, meta, opts = {})
+      time = self.class.normalize_time(time) unless opts[:normalize] == false
+
+      mute_children = meta&.delete(:mute_children)
+
+      sp = native_start_span(time, cat.to_s)
+      native_span_set_title(sp, title.to_s) if title
+      native_span_set_description(sp, desc.to_s) if desc
+      native_span_set_meta(sp, meta) if meta
+      native_span_started(sp)
+
+      @spans << sp
+      t { "started span: #{sp} - #{cat}, #{title}" }
+
+      if mute_children
+        t { "muting child instrumentation for span=#{sp}" }
+        mute_child_instrumentation(sp)
+      end
+
+      sp
+    end
+
+    def mute_child_instrumentation(span)
+      @child_instrumentation_muted_by = span
+    end
+
+    # Middleware spans that were interrupted by a throw/catch should be cached here.
+    # keys: span ids
+    # values: nsec timestamp at which the span was cached here.
+    def deferred_spans
+      @deferred_spans ||= {}
+    end
+
+    def stop(span, time)
+      t { "stopping span: #{span}" }
+
+      # If `stop` is called for a span that is not the last item in the stack,
+      # check to see if the last item has been marked as deferred. If so, close
+      # that span first, then try to close the original.
+      while deferred_spans[expected = @spans.pop]
+        normalized_stop(expected, deferred_spans.delete(expected))
+      end
+
+      handle_unexpected_stop(expected, span) unless span == expected
+
+      normalized_stop(span, time)
+      nil
+    end
+
+    def normalized_stop(span, time)
+      time = self.class.normalize_time(time)
+      native_stop_span(span, time)
+
+      if @child_instrumentation_muted_by == span
+        @child_instrumentation_muted_by = nil # restart instrumenting
       end
+    end
 
-      def use_pruning?
-        config.get(:prune_large_traces)
+    # Originally extracted from `stop`.
+    # If we attempt to close spans out of order, and it appears to be a middleware issue,
+    # disable the middleware probe and mark trace as broken.
+    def handle_unexpected_stop(expected, span)
+      message =
+        "[E0001] Spans were closed out of order. Expected to see '#{native_span_get_title(expected)}', " \
+          "but got '#{native_span_get_title(span)}' instead."
+
+      if native_span_get_category(span) == "rack.middleware" && Skylight::Probes.installed.key?(:middleware)
+        if Skylight::Probes::Middleware::Probe.disabled?
+          message << "\nWe disabled the Middleware probe but unfortunately, this didn't solve the issue."
+        else
+          Skylight::Probes::Middleware::Probe.disable!
+          message <<
+            "\n#{native_span_get_title(span)} may be a Middleware that doesn't fully conform " \
+              "to the Rack SPEC. We've disabled the Middleware probe to see if that resolves the issue."
+        end
       end
 
-      def resolve_component(component)
-        config.components[component].to_encoded_s
+      message << "\nThis request will not be tracked. Please contact support@skylight.io for more information."
+
+      error message
+
+      t { "expected=#{expected}, actual=#{span}" }
+
+      broken!
+    end
+
+    def gc_time
+      return 0 unless @gc
+
+      @gc.update
+      @gc.time
+    end
+
+    def use_pruning?
+      config.get(:prune_large_traces)
+    end
+
+    def resolve_component(component)
+      config.components[component].to_encoded_s
+    end
+
+    def component=(component)
+      resolve_component(component).tap do |c|
+        # Would it be better for the component getter to get from native?
+        @component = c
+        native_set_component(c)
       end
+    end
 
-      def component=(component)
-        resolve_component(component).tap do |c|
-          @component = c
-          native_set_component(c)
+    def preprocess_meta(meta)
+      validate_meta(meta)
+      instrumenter.extensions.trace_preprocess_meta(meta)
+    end
+
+    def validate_meta(meta)
+      unknown_keys = meta.keys - allowed_meta_keys
+      if unknown_keys.any?
+        unknown_keys.each do |key|
+          maybe_warn("unknown_meta:#{key}", "Unknown meta key will be ignored; key=#{key.inspect}")
+          meta.delete(key)
         end
       end
+    end
+
+    def allowed_meta_keys
+      META_KEYS | instrumenter.extensions.allowed_meta_keys
+    end
+
+    def maybe_warn(context, msg)
+      return if warnings_silenced?(context)
+
+      instrumenter.silence_warnings(context)
+
+      warn(msg)
+    end
+
+    def warnings_silenced?(context)
+      instrumenter.warnings_silenced?(context)
+    end
   end
 end

data/lib/skylight/user_config.rb

@@ -0,0 +1,58 @@
+require "yaml"
+require "skylight/errors"
+
+module Skylight
+  class UserConfig
+    attr_accessor :disable_dev_warning, :disable_env_warning
+
+    def initialize(config)
+      @config = config
+      @file_path = nil
+      reload
+    end
+
+    def file_path
+      return @file_path if @file_path
+
+      config_path =
+        @config[:user_config_path] ||
+          begin
+            require "etc"
+            home_dir = ENV["HOME"] || Etc.getpwuid.dir || (ENV["USER"] && File.expand_path("~#{ENV["USER"]}"))
+            if home_dir
+              File.join(home_dir, ".skylight")
+            else
+              raise ConfigError,
+                    "The Skylight `user_config_path` must be defined since the home directory cannot be inferred"
+            end
+          end
+
+      @file_path = File.expand_path(config_path)
+    end
+
+    def disable_dev_warning?
+      disable_dev_warning || ENV["SKYLIGHT_DISABLE_DEV_WARNING"] =~ /^true$/i
+    end
+
+    def disable_env_warning?
+      disable_env_warning
+    end
+
+    def reload
+      config = File.exist?(file_path) ? YAML.load_file(file_path) : false
+      return unless config
+
+      self.disable_dev_warning = !!config["disable_dev_warning"]
+      self.disable_env_warning = !!config["disable_env_warning"]
+    end
+
+    def save
+      FileUtils.mkdir_p(File.dirname(file_path))
+      File.open(file_path, "w") { |f| f.puts YAML.dump(to_hash) }
+    end
+
+    def to_hash
+      { "disable_dev_warning" => disable_dev_warning, "disable_env_warning" => disable_env_warning }
+    end
+  end
+end

data/lib/skylight/util/allocation_free.rb

@@ -0,0 +1,26 @@
+module Skylight
+  module Util
+    # Helpers to reduce memory allocation
+    module AllocationFree
+      # Find an item in an array without allocation.
+      #
+      # @param array [Array] the array to search
+      # @yield a block called against each item until a match is found
+      # @yieldparam item an item from the array
+      # @yieldreturn [Boolean] whether `item` matches the criteria
+      # return the found item or nil, if nothing found
+      def array_find(array)
+        i = 0
+
+        while i < array.size
+          item = array[i]
+          return item if yield item
+
+          i += 1
+        end
+
+        nil
+      end
+    end
+  end
+end

data/lib/skylight/util/clock.rb

@@ -0,0 +1,57 @@
+module Skylight
+  module Util
+    # A more precise clock
+    class Clock
+      def self.use_native!
+        class_eval do
+          def tick
+            native_hrtime
+          end
+        end
+      end
+
+      # rubocop:disable Lint/DuplicateMethods
+      def tick
+        now = Time.now
+        (now.to_i * 1_000_000_000) + (now.usec * 1_000)
+      end
+
+      # rubocop:enable Lint/DuplicateMethods
+
+      # TODO: rename to secs
+      def absolute_secs
+        Time.now.to_i
+      end
+
+      # TODO: remove
+      def nanos
+        tick
+      end
+
+      # TODO: remove
+      def secs
+        nanos / 1_000_000_000
+      end
+
+      class << self
+        def absolute_secs
+          default.absolute_secs
+        end
+
+        def nanos
+          default.nanos
+        end
+
+        def secs
+          default.secs
+        end
+
+        def default
+          @default ||= Clock.new
+        end
+
+        attr_writer :default
+      end
+    end
+  end
+end

data/lib/skylight/util/component.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require "uri"
 
 module Skylight
@@ -6,16 +7,17 @@ module Skylight
     class Component
       attr_accessor :environment, :name
 
-      NAME_FORMAT = /\A[a-zA-Z0-9_-]+\z/
+      NAME_FORMAT = /\A[a-zA-Z0-9_-]+\z/.freeze
       DEFAULT_NAME = "web"
       WORKER_NAME = "worker"
       DEFAULT_ENVIRONMENT = "production"
 
       def initialize(environment, name, force_worker: false)
         @environment = environment || DEFAULT_ENVIRONMENT
-        @name        = resolve_name(name, force_worker)
+        @name = resolve_name(name, force_worker)
 
         raise ArgumentError, "environment can't be blank" if @environment.empty?
+
         validate_string!(@environment, "environment")
         validate_string!(@name, "name")
       end
@@ -38,34 +40,32 @@ module Skylight
 
       # keys here should match those from the main config
       def as_json(*)
-        {
-          component: name,
-          env: environment
-        }
+        { component: name, env: environment }
       end
 
       private
 
-        def program_name
-          $PROGRAM_NAME
-        end
+      def program_name
+        $PROGRAM_NAME
+      end
 
-        def argv
-          ARGV
-        end
+      def argv
+        ARGV
+      end
 
-        def resolve_name(given_name, force_worker)
-          # don't allow workers to be called 'web'
-          return WORKER_NAME if force_worker && (given_name.nil? || given_name == DEFAULT_NAME)
-          return DEFAULT_NAME if given_name.nil?
+      def resolve_name(given_name, force_worker)
+        # don't allow workers to be called 'web'
+        return WORKER_NAME if force_worker && (given_name.nil? || given_name == DEFAULT_NAME)
+        return DEFAULT_NAME if given_name.nil?
 
-          given_name
-        end
+        given_name
+      end
+
+      def validate_string!(string, kind)
+        return true if string =~ NAME_FORMAT
 
-        def validate_string!(string, kind)
-          return true if string =~ NAME_FORMAT
-          raise ArgumentError, "#{kind} can only contain lowercase letters, numbers, and dashes"
-        end
+        raise ArgumentError, "#{kind} can only contain lowercase letters, numbers, and dashes"
+      end
     end
   end
 end

data/lib/skylight/util/deploy.rb

@@ -1,5 +1,5 @@
 require "json"
-require "skylight/core/util/logging"
+require "skylight/util/logging"
 
 module Skylight
   module Util
@@ -13,8 +13,7 @@ module Skylight
       end
 
       class EmptyDeploy
-        attr_reader :config
-        attr_reader :timestamp
+        attr_reader :config, :timestamp
 
         def initialize(config)
           @config = config
@@ -35,8 +34,8 @@ module Skylight
 
         def to_query_hash
           hash = {
-            timestamp:   timestamp,
-            deploy_id:   id.to_s[0..100] # Keep this sane
+            timestamp: timestamp,
+            deploy_id: id.to_s[0..100] # Keep this sane
           }
           hash[:git_sha] = git_sha.to_s[0..40] if git_sha # A valid SHA will never exceed 40
           hash[:description] = description[0..255] if description # Avoid massive descriptions
@@ -45,25 +44,23 @@ module Skylight
       end
 
       class DefaultDeploy < EmptyDeploy
-        include Core::Util::Logging
+        include Util::Logging
 
         def initialize(*)
           super
-          if description && !id
-            warn "The configured deploy will be ignored as an id or git_sha must be provided."
-          end
+          warn "The configured deploy will be ignored as an id or git_sha must be provided." if description && !id
         end
 
         def id
-          config.get(:'deploy.id') || git_sha
+          config.get(:"deploy.id") || git_sha
         end
 
         def git_sha
-          config.get(:'deploy.git_sha')
+          config.get(:"deploy.git_sha")
         end
 
         def description
-          config.get(:'deploy.description')
+          config.get(:"deploy.description")
         end
       end
 
@@ -87,15 +84,13 @@ module Skylight
 
         private
 
-          def get_info
-            info_path = config[:'heroku.dyno_info_path']
+        def get_info
+          info_path = config[:"heroku.dyno_info_path"]
 
-            if File.exist?(info_path)
-              if (info = JSON.parse(File.read(info_path)))
-                info["release"]
-              end
-            end
+          if File.exist?(info_path) && (info = JSON.parse(File.read(info_path)))
+            info["release"]
           end
+        end
       end
 
       class GitDeploy < EmptyDeploy
@@ -108,12 +103,12 @@ module Skylight
 
         private
 
-          def get_info
-            Dir.chdir(config.root) do
-              info = `git log -1 --pretty="%H %s" 2>&1`
-              info.split(" ", 2).map(&:strip) if $CHILD_STATUS.success?
-            end
+        def get_info
+          Dir.chdir(config.root) do
+            info = `git log -1 --pretty="%H %s" 2>&1`
+            info.split(" ", 2).map(&:strip) if $CHILD_STATUS.success?
           end
+        end
       end
 
       DEPLOY_TYPES = [DefaultDeploy, HerokuDeploy, GitDeploy].freeze