ddtrace 0.53.0 → 1.9.0

data/lib/datadog/profiling/tasks/setup.rb

@@ -0,0 +1,86 @@
+# typed: false
+
+require_relative '../../core/utils/only_once'
+require_relative '../ext/forking'
+
+module Datadog
+  module Profiling
+    module Tasks
+      # Takes care of loading our extensions/monkey patches to handle fork() and validating if CPU-time profiling is usable
+      class Setup
+        ACTIVATE_EXTENSIONS_ONLY_ONCE = Core::Utils::OnlyOnce.new
+
+        def run
+          ACTIVATE_EXTENSIONS_ONLY_ONCE.run do
+            begin
+              check_if_cpu_time_profiling_is_supported
+              activate_forking_extensions
+              setup_at_fork_hooks
+            rescue StandardError, ScriptError => e
+              Datadog.logger.warn do
+                "Profiler extensions unavailable. Cause: #{e.class.name} #{e.message} " \
+                "Location: #{Array(e.backtrace).first}"
+              end
+            end
+          end
+        end
+
+        private
+
+        def activate_forking_extensions
+          if Ext::Forking.supported?
+            Ext::Forking.apply!
+          elsif Datadog.configuration.profiling.enabled
+            Datadog.logger.debug('Profiler forking extensions skipped; forking not supported.')
+          end
+        rescue StandardError, ScriptError => e
+          Datadog.logger.warn do
+            "Profiler forking extensions unavailable. Cause: #{e.class.name} #{e.message} " \
+            "Location: #{Array(e.backtrace).first}"
+          end
+        end
+
+        def check_if_cpu_time_profiling_is_supported
+          unsupported = cpu_time_profiling_unsupported_reason
+
+          if unsupported
+            Datadog.logger.info do
+              'CPU time profiling skipped because native CPU time is not supported: ' \
+              "#{unsupported}. Profiles containing 'Wall time' data will still be reported."
+            end
+          end
+        end
+
+        def setup_at_fork_hooks
+          if Process.respond_to?(:at_fork)
+            Process.at_fork(:child) do
+              begin
+                # Restart profiler, if enabled
+                Profiling.start_if_enabled
+              rescue StandardError => e
+                Datadog.logger.warn do
+                  "Error during post-fork hooks. Cause: #{e.class.name} #{e.message} " \
+                  "Location: #{Array(e.backtrace).first}"
+                end
+              end
+            end
+          end
+        end
+
+        def cpu_time_profiling_unsupported_reason
+          # NOTE: Only the first matching reason is returned, so try to keep a nice order on reasons
+
+          if RUBY_ENGINE == 'jruby'
+            'JRuby is not supported'
+          elsif RUBY_PLATFORM.include?('darwin')
+            'Feature requires Linux; macOS is not supported'
+          elsif RUBY_PLATFORM =~ /(mswin|mingw)/
+            'Feature requires Linux; Windows is not supported'
+          elsif !RUBY_PLATFORM.include?('linux')
+            "Feature requires Linux; #{RUBY_PLATFORM} is not supported"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/profiling/trace_identifiers/ddtrace.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+# typed: true
+
+require_relative '../../tracing'
+require_relative '../../tracing/metadata/ext'
+
+module Datadog
+  module Profiling
+    module TraceIdentifiers
+      # Used by Datadog::Profiling::TraceIdentifiers::Helper to get the trace identifiers (root span id and span id)
+      # for a given thread, if there is an active trace for that thread in the supplied tracer object.
+      class Ddtrace
+        def initialize(tracer:)
+          @tracer = (tracer if tracer.respond_to?(:active_trace))
+        end
+
+        def trace_identifiers_for(thread)
+          return unless @tracer
+
+          trace = @tracer.active_trace(thread)
+          return unless trace
+
+          root_span = trace.send(:root_span)
+          span = trace.active_span
+          return unless span && root_span
+
+          root_span_id = root_span.id || 0
+          span_id = span.id || 0
+
+          [root_span_id, span_id, maybe_extract_resource(trace, root_span)] if root_span_id != 0 && span_id != 0
+        end
+
+        private
+
+        # NOTE: Currently we're only interested in HTTP service endpoints. Over time, this list may be expanded.
+        # Resources MUST NOT include personal identifiable information (PII); this should not be the case with
+        # ddtrace integrations, but worth mentioning just in case :)
+        def maybe_extract_resource(trace, root_span)
+          trace.resource if root_span.span_type == Tracing::Metadata::Ext::HTTP::TYPE_INBOUND
+        end
+      end
+    end
+  end
+end

data/lib/datadog/profiling/trace_identifiers/helper.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+# typed: true
+
+require_relative 'ddtrace'
+
+module Datadog
+  module Profiling
+    module TraceIdentifiers
+      # Helper used to retrieve the trace identifiers (root span id and span id) for a given thread,
+      # if there is an active trace for that thread for the supported tracing APIs.
+      #
+      # This data is used to connect profiles to the traces -- samples in a profile will be tagged with this data and
+      # the profile can be filtered down to look at only the samples for a given trace.
+      class Helper
+        DEFAULT_SUPPORTED_APIS = [
+          Profiling::TraceIdentifiers::Ddtrace
+        ].freeze
+        private_constant :DEFAULT_SUPPORTED_APIS
+
+        def initialize(
+          tracer:,
+          # If this is disabled, the helper will strip the optional trace_resource_container even if provided by the api
+          endpoint_collection_enabled:,
+          supported_apis: DEFAULT_SUPPORTED_APIS.map { |api| api.new(tracer: tracer) }
+        )
+          @endpoint_collection_enabled = endpoint_collection_enabled
+          @supported_apis = supported_apis
+        end
+
+        # Expected output of the #trace_identifiers_for
+        # duck type is [root_span_id, span_id, (optional trace_resource_container)]
+        def trace_identifiers_for(thread)
+          @supported_apis.each do |api|
+            trace_identifiers = api.trace_identifiers_for(thread)
+
+            if trace_identifiers
+              return @endpoint_collection_enabled ? trace_identifiers : trace_identifiers[0..1]
+            end
+          end
+
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/datadog/profiling.rb

@@ -0,0 +1,173 @@
+# typed: true
+
+require_relative 'core'
+require_relative 'core/environment/variable_helpers'
+require_relative 'core/utils/only_once'
+
+module Datadog
+  # Contains profiler for generating stack profiles, etc.
+  module Profiling
+    GOOGLE_PROTOBUF_MINIMUM_VERSION = Gem::Version.new('3.0')
+    private_constant :GOOGLE_PROTOBUF_MINIMUM_VERSION
+
+    def self.supported?
+      unsupported_reason.nil?
+    end
+
+    def self.unsupported_reason
+      # NOTE: Only the first matching reason is returned, so try to keep a nice order on reasons -- e.g. tell users
+      # first that they can't use this on JRuby before telling them that they are missing protobuf
+
+      native_library_compilation_skipped? ||
+        native_library_failed_to_load? ||
+        protobuf_gem_unavailable? ||
+        protobuf_version_unsupported? ||
+        protobuf_failed_to_load?
+    end
+
+    # Starts the profiler, if the profiler is supported by in
+    # this runtime environment and if the profiler has been enabled
+    # in configuration.
+    #
+    # @return [Boolean] `true` if the profiler has successfully started, otherwise `false`.
+    # @public_api
+    def self.start_if_enabled
+      # If the profiler was not previously touched, getting the profiler instance triggers start as a side-effect
+      # otherwise we get nil
+      profiler = Datadog.send(:components).profiler
+      # ...but we still try to start it BECAUSE if the process forks, the profiler will exist but may
+      # not yet have been started in the fork
+      profiler.start if profiler
+      !!profiler
+    end
+
+    private_class_method def self.native_library_compilation_skipped?
+      skipped_reason = try_reading_skipped_reason_file
+
+      "Your ddtrace installation is missing support for the Continuous Profiler because #{skipped_reason}" if skipped_reason
+    end
+
+    private_class_method def self.try_reading_skipped_reason_file(file_api = File)
+      # This file, if it exists, is recorded by extconf.rb during compilation of the native extension
+      skipped_reason_file = "#{__dir__}/../../ext/ddtrace_profiling_native_extension/skipped_reason.txt"
+
+      begin
+        return unless file_api.exist?(skipped_reason_file)
+
+        contents = file_api.read(skipped_reason_file).strip
+        contents unless contents.empty?
+      rescue StandardError
+        # Do nothing
+      end
+    end
+
+    private_class_method def self.protobuf_gem_unavailable?
+      # NOTE: On environments where protobuf is already loaded, we skip the check. This allows us to support environments
+      # where no Gem.loaded_version is NOT available but customers are able to load protobuf; see for instance
+      # https://github.com/teamcapybara/capybara/commit/caf3bcd7664f4f2691d0ca9ef3be9a2a954fecfb
+      if !defined?(::Google::Protobuf) && Gem.loaded_specs['google-protobuf'].nil?
+        "Missing google-protobuf dependency; please add `gem 'google-protobuf', '~> 3.0'` to your Gemfile or gems.rb file"
+      end
+    end
+
+    private_class_method def self.protobuf_version_unsupported?
+      # See above for why we skip the check when protobuf is already loaded; note that when protobuf was already loaded
+      # we skip the version check to avoid the call to Gem.loaded_specs. Unfortunately, protobuf does not seem to
+      # expose the gem version constant elsewhere, so in that setup we are not able to check the version.
+      if !defined?(::Google::Protobuf) && Gem.loaded_specs['google-protobuf'].version < GOOGLE_PROTOBUF_MINIMUM_VERSION
+        'Your google-protobuf is too old; ensure that you have google-protobuf >= 3.0 by ' \
+        "adding `gem 'google-protobuf', '~> 3.0'` to your Gemfile or gems.rb file"
+      end
+    end
+
+    private_class_method def self.protobuf_failed_to_load?
+      unless protobuf_loaded_successfully?
+        'There was an error loading the google-protobuf library; see previous warning message for details'
+      end
+    end
+
+    # The `google-protobuf` gem depends on a native component, and its creators helpfully tried to provide precompiled
+    # versions of this extension on rubygems.org.
+    #
+    # Unfortunately, for a long time, the supported Ruby versions metadata on these precompiled versions of the extension
+    # was not correctly set. (This is fixed in newer versions -- but not all Ruby versions we want to support can use
+    # these.)
+    #
+    # Thus, the gem can still be installed, but can be in a broken state. To avoid breaking customer applications, we
+    # use this helper to load it and gracefully handle failures.
+    private_class_method def self.protobuf_loaded_successfully?
+      return @protobuf_loaded if defined?(@protobuf_loaded)
+
+      begin
+        require 'google/protobuf'
+        @protobuf_loaded = true
+      rescue LoadError => e
+        # NOTE: We use Kernel#warn here because this code gets run BEFORE Datadog.logger is actually set up.
+        # In the future it'd be nice to shuffle the logger startup to happen first to avoid this special case.
+        Kernel.warn(
+          '[DDTRACE] Error while loading google-protobuf gem. ' \
+          "Cause: '#{e.class.name} #{e.message}' Location: '#{Array(e.backtrace).first}'. " \
+          'This can happen when google-protobuf is missing its native components. ' \
+          'To fix this, try removing and reinstalling the gem, forcing it to recompile the components: ' \
+          '`gem uninstall google-protobuf -a; BUNDLE_FORCE_RUBY_PLATFORM=true bundle install`. ' \
+          'If the error persists, please contact Datadog support at <https://docs.datadoghq.com/help/>.'
+        )
+        @protobuf_loaded = false
+      end
+    end
+
+    private_class_method def self.native_library_failed_to_load?
+      success, exception = try_loading_native_library
+
+      unless success
+        if exception
+          'There was an error loading the profiling native extension due to ' \
+          "'#{exception.class.name} #{exception.message}' at '#{exception.backtrace.first}'"
+        else
+          'The profiling native extension did not load correctly. ' \
+          'For help solving this issue, please contact Datadog support at <https://docs.datadoghq.com/help/>.' \
+        end
+      end
+    end
+
+    private_class_method def self.try_loading_native_library
+      begin
+        require_relative 'profiling/load_native_extension'
+
+        success =
+          defined?(Profiling::NativeExtension) && Profiling::NativeExtension.send(:native_working?)
+        [success, nil]
+      rescue StandardError, LoadError => e
+        [false, e]
+      end
+    end
+
+    private_class_method def self.load_profiling
+      return false unless supported?
+
+      require_relative 'profiling/ext/forking'
+      require_relative 'profiling/collectors/code_provenance'
+      require_relative 'profiling/collectors/cpu_and_wall_time'
+      require_relative 'profiling/collectors/cpu_and_wall_time_worker'
+      require_relative 'profiling/collectors/dynamic_sampling_rate'
+      require_relative 'profiling/collectors/idle_sampling_helper'
+      require_relative 'profiling/collectors/old_stack'
+      require_relative 'profiling/collectors/stack'
+      require_relative 'profiling/stack_recorder'
+      require_relative 'profiling/old_recorder'
+      require_relative 'profiling/exporter'
+      require_relative 'profiling/scheduler'
+      require_relative 'profiling/tasks/setup'
+      require_relative 'profiling/profiler'
+      require_relative 'profiling/native_extension'
+      require_relative 'profiling/trace_identifiers/helper'
+      require_relative 'profiling/pprof/pprof_pb'
+      require_relative 'profiling/tag_builder'
+      require_relative 'profiling/http_transport'
+
+      true
+    end
+
+    load_profiling
+  end
+end

data/lib/datadog/tracing/analytics.rb

@@ -0,0 +1,25 @@
+# typed: true
+
+require_relative 'metadata/ext'
+
+module Datadog
+  module Tracing
+    # Defines analytics behavior
+    module Analytics
+      class << self
+        def set_sample_rate(span_op, sample_rate)
+          return if span_op.nil? || !sample_rate.is_a?(Numeric)
+
+          span_op.set_metric(Metadata::Ext::Analytics::TAG_SAMPLE_RATE, sample_rate)
+        end
+
+        def set_measured(span_op, value = true)
+          return if span_op.nil?
+
+          value = value == true || value == 1 ? 1 : 0 # rubocop:disable Style/MultipleComparison
+          span_op.set_metric(Metadata::Ext::Analytics::TAG_MEASURED, value)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/buffer.rb

@@ -0,0 +1,132 @@
+# typed: true
+
+require_relative '../core'
+require_relative '../core/environment/ext'
+require_relative '../core/buffer/thread_safe'
+require_relative '../core/buffer/cruby'
+require_relative '../core/diagnostics/health'
+
+module Datadog
+  module Tracing
+    # Health metrics for trace buffers.
+    module MeasuredBuffer
+      include Kernel # Ensure that kernel methods are always available (https://sorbet.org/docs/error-reference#7003)
+
+      def initialize(*_)
+        super
+
+        @buffer_accepted = 0
+        @buffer_accepted_lengths = 0
+        @buffer_dropped = 0
+        @buffer_spans = 0
+      end
+
+      def add!(trace)
+        super
+
+        # Emit health metrics
+        measure_accept(trace)
+      end
+
+      def add_all!(traces)
+        super
+
+        # Emit health metrics
+        traces.each { |trace| measure_accept(trace) }
+      end
+
+      def replace!(trace)
+        discarded_trace = super
+
+        # Emit health metrics
+        measure_accept(trace)
+        measure_drop(discarded_trace) if discarded_trace
+
+        discarded_trace
+      end
+
+      # Stored traces are returned and the local buffer is reset.
+      def drain!
+        traces = super
+        measure_pop(traces)
+        traces
+      end
+
+      def measure_accept(trace)
+        @buffer_accepted += 1
+        @buffer_accepted_lengths += trace.length
+
+        @buffer_spans += trace.length
+      rescue StandardError => e
+        Datadog.logger.debug(
+          "Failed to measure queue accept. Cause: #{e.class.name} #{e.message} Source: #{Array(e.backtrace).first}"
+        )
+      end
+
+      def measure_drop(trace)
+        @buffer_dropped += 1
+
+        @buffer_spans -= trace.length
+      rescue StandardError => e
+        Datadog.logger.debug(
+          "Failed to measure queue drop. Cause: #{e.class.name} #{e.message} Source: #{Array(e.backtrace).first}"
+        )
+      end
+
+      def measure_pop(traces)
+        # Accepted, cumulative totals
+        Datadog.health_metrics.queue_accepted(@buffer_accepted)
+        Datadog.health_metrics.queue_accepted_lengths(@buffer_accepted_lengths)
+
+        # Dropped, cumulative totals
+        Datadog.health_metrics.queue_dropped(@buffer_dropped)
+        # TODO: are we missing a +queue_dropped_lengths+ metric?
+
+        # Queue gauges, current values
+        Datadog.health_metrics.queue_max_length(@max_size)
+        Datadog.health_metrics.queue_spans(@buffer_spans)
+        Datadog.health_metrics.queue_length(traces.length)
+
+        # Reset aggregated metrics
+        @buffer_accepted = 0
+        @buffer_accepted_lengths = 0
+        @buffer_dropped = 0
+        @buffer_spans = 0
+      rescue StandardError => e
+        Datadog.logger.debug(
+          "Failed to measure queue. Cause: #{e.class.name} #{e.message} Source: #{Array(e.backtrace).first}"
+        )
+      end
+    end
+
+    # Trace buffer that stores application traces, has a maximum size, and
+    # can be safely used concurrently on any environment.
+    #
+    # @see Datadog::Core::Buffer::ThreadSafe
+    class ThreadSafeTraceBuffer < Core::Buffer::ThreadSafe
+      prepend MeasuredBuffer
+    end
+
+    # Trace buffer that stores application traces, has a maximum size, and
+    # can be safely used concurrently with CRuby.
+    #
+    # @see Datadog::Core::Buffer::CRuby
+    class CRubyTraceBuffer < Core::Buffer::CRuby
+      prepend MeasuredBuffer
+    end
+
+    # Trace buffer that stores application traces. The buffer has a maximum size and when
+    # the buffer is full, a random trace is discarded. This class is thread-safe and is used
+    # automatically by the ``Tracer`` instance when a ``Span`` is finished.
+    #
+    # We choose the default TraceBuffer implementation for current platform dynamically here.
+    #
+    # TODO We should restructure this module, so that classes are not declared at top-level ::Datadog.
+    # TODO Making such a change is potentially breaking for users manually configuring the tracer.
+    TraceBuffer = if Core::Environment::Ext::RUBY_ENGINE == 'ruby'
+                    CRubyTraceBuffer
+                  else
+                    ThreadSafeTraceBuffer
+                  end
+  end
+end

data/lib/datadog/tracing/client_ip.rb

@@ -0,0 +1,164 @@
+# typed: true
+
+require_relative '../core/configuration'
+require_relative 'metadata/ext'
+require_relative 'span'
+
+require 'ipaddr'
+
+module Datadog
+  module Tracing
+    # Common functions for supporting the `http.client_ip` span attribute.
+    module ClientIp
+      DEFAULT_IP_HEADERS_NAMES = %w[
+        x-forwarded-for
+        x-real-ip
+        x-client-ip
+        x-forwarded
+        x-cluster-client-ip
+        forwarded-for
+        forwarded
+        via
+        true-client-ip
+      ].freeze
+
+      TAG_MULTIPLE_IP_HEADERS = '_dd.multiple-ip-headers'.freeze
+
+      # Sets the `http.client_ip` tag on the given span.
+      #
+      # This function respects the user's settings: if they disable the client IP tagging,
+      #   or provide a different IP header name.
+      #
+      # If multiple IP headers are present in the request, this function will instead set
+      #   the `_dd.multiple-ip-headers` tag with the names of the present headers,
+      #   and **NOT** set the `http.client_ip` tag.
+      #
+      # @param [Span] span The span that's associated with the request.
+      # @param [HeaderCollection, #get, nil] headers A collection with the request headers.
+      # @param [String, nil] remote_ip The remote IP the request associated with the span is sent to.
+      def self.set_client_ip_tag(span, headers: nil, remote_ip: nil)
+        return unless configuration.enabled
+
+        set_client_ip_tag!(span, headers: headers, remote_ip: remote_ip)
+      end
+
+      # Forcefully sets the `http.client_ip` tag on the given span.
+      #
+      # This function ignores the user's `enabled` setting.
+      #
+      # @param [Span] span The span that's associated with the request.
+      # @param [HeaderCollection, #get, nil] headers A collection with the request headers.
+      # @param [String, nil] remote_ip The remote IP the request associated with the span is sent to.
+      def self.set_client_ip_tag!(span, headers: nil, remote_ip: nil)
+        result = raw_ip_from_request(headers, remote_ip)
+
+        if result.raw_ip
+          ip = strip_decorations(result.raw_ip)
+          return unless valid_ip?(ip)
+
+          span.set_tag(Tracing::Metadata::Ext::HTTP::TAG_CLIENT_IP, ip)
+        elsif result.multiple_ip_headers
+          span.set_tag(TAG_MULTIPLE_IP_HEADERS, result.multiple_ip_headers.keys.join(','))
+        end
+      end
+
+      IpExtractionResult = Struct.new(:raw_ip, :multiple_ip_headers)
+
+      # Returns a result struct that holds the raw client IP associated with the request if it was
+      #   retrieved successfully.
+      #
+      # The client IP is looked up by the following logic:
+      # * If the user has configured a header name, return that header's value.
+      # * If exactly one of the known IP headers is present, return that header's value.
+      # * If none of the known IP headers are present, return the remote IP from the request.
+      #
+      # If more than one of the known IP headers is present, the result will have a `multiple_ip_headers`
+      #   field with the name of the present IP headers.
+      #
+      # @param [Datadog::Core::HeaderCollection, #get, nil] headers The request headers
+      # @param [String] remote_ip The remote IP of the request.
+      # @return [IpExtractionResult] A struct that holds the unprocessed IP value,
+      #   or `nil` if it wasn't found. Additionally, the `multiple_ip_headers` fields will hold the
+      #   name of known IP headers present in the request if more than one of these were found.
+      def self.raw_ip_from_request(headers, remote_ip)
+        return IpExtractionResult.new(headers && headers.get(configuration.header_name), nil) if configuration.header_name
+
+        headers_present = ip_headers(headers)
+
+        case headers_present.size
+        when 0
+          IpExtractionResult.new(remote_ip, nil)
+        when 1
+          IpExtractionResult.new(headers_present.values.first, nil)
+        else
+          IpExtractionResult.new(nil, headers_present)
+        end
+      end
+
+      # Removes any port notations or zone specifiers from the IP address without
+      #   verifying its validity.
+      def self.strip_decorations(address)
+        return strip_ipv4_port(address) if likely_ipv4?(address)
+
+        address = strip_ipv6_port(address)
+
+        strip_zone_specifier(address)
+      end
+
+      def self.strip_zone_specifier(ipv6)
+        ipv6.gsub(/%.*/, '')
+      end
+
+      def self.strip_ipv4_port(ip)
+        ip.gsub(/:\d+\z/, '')
+      end
+
+      def self.strip_ipv6_port(ip)
+        if /\[(.*)\](?::\d+)?/ =~ ip
+          Regexp.last_match(1)
+        else
+          ip
+        end
+      end
+
+      # Returns whether the given value is more likely to be an IPv4 than an IPv6 address.
+      #
+      # This is done by checking if a dot (`'.'`) character appears before a colon (`':'`) in the value.
+      # The rationale is that in valid IPv6 addresses, colons will always preced dots,
+      #   and in valid IPv4 addresses dots will always preced colons.
+      def self.likely_ipv4?(value)
+        dot_index = value.index('.') || value.size
+        colon_index = value.index(':') || value.size
+
+        dot_index < colon_index
+      end
+
+      # Determines whether the given string is a valid IPv4 or IPv6 address.
+      def self.valid_ip?(ip)
+        # Client IPs should not have subnet masks even though IPAddr can parse them.
+        return false if ip.include?('/')
+
+        begin
+          IPAddr.new(ip)
+
+          true
+        rescue IPAddr::Error
+          false
+        end
+      end
+
+      def self.ip_headers(headers)
+        return {} unless headers
+
+        DEFAULT_IP_HEADERS_NAMES.each_with_object({}) do |name, result|
+          value = headers.get(name)
+          result[name] = value unless value.nil?
+        end
+      end
+
+      def self.configuration
+        Datadog.configuration.tracing.client_ip
+      end
+    end
+  end
+end