ddtrace 1.1.0 → 1.9.0

data/lib/datadog/profiling/scheduler.rb

@@ -1,42 +1,41 @@
 # typed: true
 
-require 'datadog/core/utils/time'
+require_relative '../core/utils/time'
 
-require 'datadog/core/worker'
-require 'datadog/core/workers/polling'
+require_relative '../core/worker'
+require_relative '../core/workers/polling'
 
 module Datadog
   module Profiling
-    # Periodically (every DEFAULT_INTERVAL_SECONDS) takes data from the `Recorder` and pushes them to all configured
-    # `Exporter`s. Runs on its own background thread.
+    # Periodically (every DEFAULT_INTERVAL_SECONDS) takes a profile from the `Exporter` and reports it using the
+    # configured transport. Runs on its own background thread.
     class Scheduler < Core::Worker
       include Core::Workers::Polling
 
       DEFAULT_INTERVAL_SECONDS = 60
       MINIMUM_INTERVAL_SECONDS = 0
 
-      # Profiles with duration less than this will not be reported
-      PROFILE_DURATION_THRESHOLD_SECONDS = 1
-
       # We sleep for at most this duration seconds before reporting data to avoid multi-process applications all
       # reporting profiles at the exact same time
       DEFAULT_FLUSH_JITTER_MAXIMUM_SECONDS = 3
 
-      private_constant :DEFAULT_INTERVAL_SECONDS, :MINIMUM_INTERVAL_SECONDS, :PROFILE_DURATION_THRESHOLD_SECONDS
+      private
 
       attr_reader \
-        :exporters,
-        :recorder
+        :exporter,
+        :transport
+
+      public
 
       def initialize(
-        recorder,
-        exporters,
+        exporter:,
+        transport:,
         fork_policy: Core::Workers::Async::Thread::FORK_POLICY_RESTART, # Restart in forks by default
         interval: DEFAULT_INTERVAL_SECONDS,
         enabled: true
       )
-        @recorder = recorder
-        @exporters = [exporters].flatten
+        @exporter = exporter
+        @transport = transport
 
         # Workers::Async::Thread settings
         self.fork_policy = fork_policy
@@ -67,24 +66,21 @@ module Datadog
         end
       end
 
-      def after_fork
-        # Clear recorder's buffers by flushing events.
-        # Objects from parent process will copy-on-write,
-        # and we don't want to send events for the wrong process.
-        recorder.flush
-      end
-
       # Configure Workers::IntervalLoop to not report immediately when scheduler starts
       #
       # When a scheduler gets created (or reset), we don't want it to immediately try to flush; we want it to wait for
       # the loop wait time first. This avoids an issue where the scheduler reported a mostly-empty profile if the
-      # application just started but this thread took a bit longer so there's already samples in the recorder.
+      # application just started but this thread took a bit longer so there's already profiling data in the exporter.
       def loop_wait_before_first_iteration?
         true
       end
 
       def work_pending?
-        !recorder.empty?
+        exporter.can_flush?
+      end
+
+      def reset_after_fork
+        exporter.reset_after_fork
       end
 
       private
@@ -100,16 +96,10 @@ module Datadog
       end
 
       def flush_events
-        # Get events from recorder
-        flush = recorder.flush
+        # Collect data to be exported
+        flush = exporter.flush
 
-        if duration_below_threshold?(flush)
-          Datadog.logger.debug do
-            "Skipped exporting profiling events as profile duration is below minimum (#{flush.event_count} events skipped)"
-          end
-
-          return flush
-        end
+        return false unless flush
 
         # Sleep for a bit to cause misalignment between profilers in multi-process applications
         #
@@ -127,24 +117,13 @@ module Datadog
           sleep(jitter_seconds)
         end
 
-        # Send events to each exporter
-        if flush.event_count > 0
-          exporters.each do |exporter|
-            begin
-              exporter.export(flush)
-            rescue StandardError => e
-              Datadog.logger.error(
-                "Unable to export #{flush.event_count} profiling events. Cause: #{e} Location: #{Array(e.backtrace).first}"
-              )
-            end
-          end
+        begin
+          transport.export(flush)
+        rescue StandardError => e
+          Datadog.logger.error("Unable to report profile. Cause: #{e} Location: #{Array(e.backtrace).first}")
         end
 
-        flush
-      end
-
-      def duration_below_threshold?(flush)
-        (flush.finish - flush.start) < PROFILE_DURATION_THRESHOLD_SECONDS
+        true
       end
     end
   end

data/lib/datadog/profiling/stack_recorder.rb

@@ -2,11 +2,22 @@
 
 module Datadog
   module Profiling
-    # Used to wrap a ddprof_ffi_Profile in a Ruby object and expose Ruby-level serialization APIs
+    # Stores stack samples in a native libdatadog data structure and expose Ruby-level serialization APIs
+    # Note that `record_sample` is only accessible from native code.
     # Methods prefixed with _native_ are implemented in `stack_recorder.c`
     class StackRecorder
+      def initialize
+        # This mutex works in addition to the fancy C-level mutexes we have in the native side (see the docs there).
+        # It prevents multiple Ruby threads calling serialize at the same time -- something like
+        # `10.times { Thread.new { stack_recorder.serialize } }`.
+        # This isn't something we expect to happen normally, but because it would break the assumptions of the
+        # C-level mutexes (that there is a single serializer thread), we add it here as an extra safeguard against it
+        # accidentally happening.
+        @no_concurrent_synchronize_mutex = Mutex.new
+      end
+
       def serialize
-        status, result = self.class._native_serialize(self)
+        status, result = @no_concurrent_synchronize_mutex.synchronize { self.class._native_serialize(self) }
 
         if status == :ok
           start, finish, encoded_pprof = result
@@ -23,11 +34,46 @@ module Datadog
         end
       end
 
-      # Used only for Ruby 2.2 and below which don't have the native `rb_time_timespec_new` API
-      # Called from native code
+      def serialize!
+        status, result = @no_concurrent_synchronize_mutex.synchronize { self.class._native_serialize(self) }
+
+        if status == :ok
+          _start, _finish, encoded_pprof = result
+
+          encoded_pprof
+        else
+          error_message = result
+
+          raise("Failed to serialize profiling data: #{error_message}")
+        end
+      end
+
+      def clear
+        status, result = @no_concurrent_synchronize_mutex.synchronize { self.class._native_clear(self) }
+
+        if status == :ok
+          finish_timestamp = result
+
+          Datadog.logger.debug { "Cleared profile at #{finish_timestamp}" }
+
+          finish_timestamp
+        else
+          error_message = result
+
+          Datadog.logger.error("Failed to clear profiling data: #{error_message}")
+
+          nil
+        end
+      end
+
+      # Used only for Ruby 2.2 which doesn't have the native `rb_time_timespec_new` API; called from native code
       def self.ruby_time_from(timespec_seconds, timespec_nanoseconds)
         Time.at(0).utc + timespec_seconds + (timespec_nanoseconds.to_r / 1_000_000_000)
       end
+
+      def reset_after_fork
+        self.class._native_reset_after_fork(self)
+      end
     end
   end
 end

data/lib/datadog/profiling/tag_builder.rb

@@ -1,5 +1,7 @@
 # typed: true
 
+require_relative '../core/utils'
+
 module Datadog
   module Profiling
     # Builds a hash of default plus user tags to be included in a profile
@@ -41,7 +43,10 @@ module Datadog
         tags[FORM_FIELD_TAG_SERVICE] = service if service
         tags[FORM_FIELD_TAG_VERSION] = version if version
 
-        user_tags.merge(tags)
+        # Make sure everything is an utf-8 string, to avoid encoding issues in native code/libddprof/further downstream
+        user_tags.merge(tags).map do |key, value|
+          [Datadog::Core::Utils.utf8_encode(key), Datadog::Core::Utils.utf8_encode(value)]
+        end.to_h
       end
     end
   end

data/lib/datadog/profiling/tasks/setup.rb

@@ -1,7 +1,7 @@
 # typed: false
 
-require 'datadog/core/utils/only_once'
-require 'datadog/profiling/ext/forking'
+require_relative '../../core/utils/only_once'
+require_relative '../ext/forking'
 
 module Datadog
   module Profiling
@@ -55,13 +55,6 @@ module Datadog
           if Process.respond_to?(:at_fork)
             Process.at_fork(:child) do
               begin
-                # When Ruby forks, clock IDs for each of the threads
-                # will change. We can only update these IDs from the
-                # execution context of the thread that owns it.
-                # This hook will update the IDs for the main thread
-                # after a fork occurs.
-                Thread.current.send(:update_native_ids) if Thread.current.respond_to?(:update_native_ids, true)
-
                 # Restart profiler, if enabled
                 Profiling.start_if_enabled
               rescue StandardError => e

data/lib/datadog/profiling/trace_identifiers/ddtrace.rb

@@ -2,8 +2,8 @@
 
 # typed: true
 
-require 'datadog/tracing'
-require 'datadog/tracing/metadata/ext'
+require_relative '../../tracing'
+require_relative '../../tracing/metadata/ext'
 
 module Datadog
   module Profiling

data/lib/datadog/profiling/trace_identifiers/helper.rb

@@ -2,7 +2,7 @@
 
 # typed: true
 
-require 'datadog/profiling/trace_identifiers/ddtrace'
+require_relative 'ddtrace'
 
 module Datadog
   module Profiling

data/lib/datadog/profiling.rb

@@ -1,12 +1,12 @@
 # typed: true
 
-require 'datadog/core'
-require 'datadog/core/environment/variable_helpers'
-require 'datadog/core/utils/only_once'
+require_relative 'core'
+require_relative 'core/environment/variable_helpers'
+require_relative 'core/utils/only_once'
 
 module Datadog
   # Contains profiler for generating stack profiles, etc.
-  module Profiling # rubocop:disable Metrics/ModuleLength
+  module Profiling
     GOOGLE_PROTOBUF_MINIMUM_VERSION = Gem::Version.new('3.0')
     private_constant :GOOGLE_PROTOBUF_MINIMUM_VERSION
 
@@ -132,7 +132,7 @@ module Datadog
 
     private_class_method def self.try_loading_native_library
       begin
-        require 'datadog/profiling/load_native_extension'
+        require_relative 'profiling/load_native_extension'
 
         success =
           defined?(Profiling::NativeExtension) && Profiling::NativeExtension.send(:native_working?)
@@ -145,22 +145,25 @@ module Datadog
     private_class_method def self.load_profiling
       return false unless supported?
 
-      require 'datadog/profiling/ext/forking'
-      require 'datadog/profiling/collectors/code_provenance'
-      require 'datadog/profiling/collectors/old_stack'
-      require 'datadog/profiling/collectors/stack'
-      require 'datadog/profiling/stack_recorder'
-      require 'datadog/profiling/exporter'
-      require 'datadog/profiling/recorder'
-      require 'datadog/profiling/scheduler'
-      require 'datadog/profiling/tasks/setup'
-      require 'datadog/profiling/transport/io'
-      require 'datadog/profiling/transport/http'
-      require 'datadog/profiling/profiler'
-      require 'datadog/profiling/native_extension'
-      require 'datadog/profiling/trace_identifiers/helper'
-      require 'datadog/profiling/pprof/pprof_pb'
-      require 'datadog/profiling/tag_builder'
+      require_relative 'profiling/ext/forking'
+      require_relative 'profiling/collectors/code_provenance'
+      require_relative 'profiling/collectors/cpu_and_wall_time'
+      require_relative 'profiling/collectors/cpu_and_wall_time_worker'
+      require_relative 'profiling/collectors/dynamic_sampling_rate'
+      require_relative 'profiling/collectors/idle_sampling_helper'
+      require_relative 'profiling/collectors/old_stack'
+      require_relative 'profiling/collectors/stack'
+      require_relative 'profiling/stack_recorder'
+      require_relative 'profiling/old_recorder'
+      require_relative 'profiling/exporter'
+      require_relative 'profiling/scheduler'
+      require_relative 'profiling/tasks/setup'
+      require_relative 'profiling/profiler'
+      require_relative 'profiling/native_extension'
+      require_relative 'profiling/trace_identifiers/helper'
+      require_relative 'profiling/pprof/pprof_pb'
+      require_relative 'profiling/tag_builder'
+      require_relative 'profiling/http_transport'
 
       true
     end

data/lib/datadog/tracing/analytics.rb

@@ -1,6 +1,6 @@
 # typed: true
 
-require 'datadog/tracing/metadata/ext'
+require_relative 'metadata/ext'
 
 module Datadog
   module Tracing

data/lib/datadog/tracing/buffer.rb

@@ -1,10 +1,10 @@
 # typed: true
 
-require 'datadog/core'
-require 'datadog/core/environment/ext'
-require 'datadog/core/buffer/thread_safe'
-require 'datadog/core/buffer/cruby'
-require 'datadog/core/diagnostics/health'
+require_relative '../core'
+require_relative '../core/environment/ext'
+require_relative '../core/buffer/thread_safe'
+require_relative '../core/buffer/cruby'
+require_relative '../core/diagnostics/health'
 
 module Datadog
   module Tracing

data/lib/datadog/tracing/client_ip.rb

@@ -0,0 +1,164 @@
+# typed: true
+
+require_relative '../core/configuration'
+require_relative 'metadata/ext'
+require_relative 'span'
+
+require 'ipaddr'
+
+module Datadog
+  module Tracing
+    # Common functions for supporting the `http.client_ip` span attribute.
+    module ClientIp
+      DEFAULT_IP_HEADERS_NAMES = %w[
+        x-forwarded-for
+        x-real-ip
+        x-client-ip
+        x-forwarded
+        x-cluster-client-ip
+        forwarded-for
+        forwarded
+        via
+        true-client-ip
+      ].freeze
+
+      TAG_MULTIPLE_IP_HEADERS = '_dd.multiple-ip-headers'.freeze
+
+      # Sets the `http.client_ip` tag on the given span.
+      #
+      # This function respects the user's settings: if they disable the client IP tagging,
+      #   or provide a different IP header name.
+      #
+      # If multiple IP headers are present in the request, this function will instead set
+      #   the `_dd.multiple-ip-headers` tag with the names of the present headers,
+      #   and **NOT** set the `http.client_ip` tag.
+      #
+      # @param [Span] span The span that's associated with the request.
+      # @param [HeaderCollection, #get, nil] headers A collection with the request headers.
+      # @param [String, nil] remote_ip The remote IP the request associated with the span is sent to.
+      def self.set_client_ip_tag(span, headers: nil, remote_ip: nil)
+        return unless configuration.enabled
+
+        set_client_ip_tag!(span, headers: headers, remote_ip: remote_ip)
+      end
+
+      # Forcefully sets the `http.client_ip` tag on the given span.
+      #
+      # This function ignores the user's `enabled` setting.
+      #
+      # @param [Span] span The span that's associated with the request.
+      # @param [HeaderCollection, #get, nil] headers A collection with the request headers.
+      # @param [String, nil] remote_ip The remote IP the request associated with the span is sent to.
+      def self.set_client_ip_tag!(span, headers: nil, remote_ip: nil)
+        result = raw_ip_from_request(headers, remote_ip)
+
+        if result.raw_ip
+          ip = strip_decorations(result.raw_ip)
+          return unless valid_ip?(ip)
+
+          span.set_tag(Tracing::Metadata::Ext::HTTP::TAG_CLIENT_IP, ip)
+        elsif result.multiple_ip_headers
+          span.set_tag(TAG_MULTIPLE_IP_HEADERS, result.multiple_ip_headers.keys.join(','))
+        end
+      end
+
+      IpExtractionResult = Struct.new(:raw_ip, :multiple_ip_headers)
+
+      # Returns a result struct that holds the raw client IP associated with the request if it was
+      #   retrieved successfully.
+      #
+      # The client IP is looked up by the following logic:
+      # * If the user has configured a header name, return that header's value.
+      # * If exactly one of the known IP headers is present, return that header's value.
+      # * If none of the known IP headers are present, return the remote IP from the request.
+      #
+      # If more than one of the known IP headers is present, the result will have a `multiple_ip_headers`
+      #   field with the name of the present IP headers.
+      #
+      # @param [Datadog::Core::HeaderCollection, #get, nil] headers The request headers
+      # @param [String] remote_ip The remote IP of the request.
+      # @return [IpExtractionResult] A struct that holds the unprocessed IP value,
+      #   or `nil` if it wasn't found. Additionally, the `multiple_ip_headers` fields will hold the
+      #   name of known IP headers present in the request if more than one of these were found.
+      def self.raw_ip_from_request(headers, remote_ip)
+        return IpExtractionResult.new(headers && headers.get(configuration.header_name), nil) if configuration.header_name
+
+        headers_present = ip_headers(headers)
+
+        case headers_present.size
+        when 0
+          IpExtractionResult.new(remote_ip, nil)
+        when 1
+          IpExtractionResult.new(headers_present.values.first, nil)
+        else
+          IpExtractionResult.new(nil, headers_present)
+        end
+      end
+
+      # Removes any port notations or zone specifiers from the IP address without
+      #   verifying its validity.
+      def self.strip_decorations(address)
+        return strip_ipv4_port(address) if likely_ipv4?(address)
+
+        address = strip_ipv6_port(address)
+
+        strip_zone_specifier(address)
+      end
+
+      def self.strip_zone_specifier(ipv6)
+        ipv6.gsub(/%.*/, '')
+      end
+
+      def self.strip_ipv4_port(ip)
+        ip.gsub(/:\d+\z/, '')
+      end
+
+      def self.strip_ipv6_port(ip)
+        if /\[(.*)\](?::\d+)?/ =~ ip
+          Regexp.last_match(1)
+        else
+          ip
+        end
+      end
+
+      # Returns whether the given value is more likely to be an IPv4 than an IPv6 address.
+      #
+      # This is done by checking if a dot (`'.'`) character appears before a colon (`':'`) in the value.
+      # The rationale is that in valid IPv6 addresses, colons will always preced dots,
+      #   and in valid IPv4 addresses dots will always preced colons.
+      def self.likely_ipv4?(value)
+        dot_index = value.index('.') || value.size
+        colon_index = value.index(':') || value.size
+
+        dot_index < colon_index
+      end
+
+      # Determines whether the given string is a valid IPv4 or IPv6 address.
+      def self.valid_ip?(ip)
+        # Client IPs should not have subnet masks even though IPAddr can parse them.
+        return false if ip.include?('/')
+
+        begin
+          IPAddr.new(ip)
+
+          true
+        rescue IPAddr::Error
+          false
+        end
+      end
+
+      def self.ip_headers(headers)
+        return {} unless headers
+
+        DEFAULT_IP_HEADERS_NAMES.each_with_object({}) do |name, result|
+          value = headers.get(name)
+          result[name] = value unless value.nil?
+        end
+      end
+
+      def self.configuration
+        Datadog.configuration.tracing.client_ip
+      end
+    end
+  end
+end

data/lib/datadog/tracing/configuration/ext.rb

@@ -3,7 +3,11 @@
 module Datadog
   module Tracing
     module Configuration
+      # Constants for configuration settings
+      # e.g. Env vars, default values, enums, etc...
       module Ext
+        ENV_ENABLED = 'DD_TRACE_ENABLED'.freeze
+
         # @public_api
         module Analytics
           ENV_TRACE_ANALYTICS_ENABLED = 'DD_TRACE_ANALYTICS_ENABLED'.freeze
@@ -16,11 +20,38 @@ module Datadog
 
         # @public_api
         module Distributed
+          # Custom Datadog format
           PROPAGATION_STYLE_DATADOG = 'Datadog'.freeze
+
+          PROPAGATION_STYLE_B3_MULTI_HEADER = 'b3multi'.freeze
+          # @deprecated Use `b3multi` instead.
           PROPAGATION_STYLE_B3 = 'B3'.freeze
-          PROPAGATION_STYLE_B3_SINGLE_HEADER = 'B3 single header'.freeze
-          ENV_PROPAGATION_STYLE_INJECT = 'DD_PROPAGATION_STYLE_INJECT'.freeze
-          ENV_PROPAGATION_STYLE_EXTRACT = 'DD_PROPAGATION_STYLE_EXTRACT'.freeze
+
+          PROPAGATION_STYLE_B3_SINGLE_HEADER = 'b3'.freeze
+          # @deprecated Use `b3` instead.
+          PROPAGATION_STYLE_B3_SINGLE_HEADER_OLD = 'B3 single header'.freeze
+
+          # W3C Trace Context
+          PROPAGATION_STYLE_TRACE_CONTEXT = 'tracecontext'.freeze
+
+          # Sets both extract and inject propagation style tho the provided value.
+          # Has lower precedence than `DD_TRACE_PROPAGATION_STYLE_INJECT` or
+          # `DD_TRACE_PROPAGATION_STYLE_EXTRACT`.
+          ENV_PROPAGATION_STYLE = 'DD_TRACE_PROPAGATION_STYLE'.freeze
+
+          ENV_PROPAGATION_STYLE_INJECT = 'DD_TRACE_PROPAGATION_STYLE_INJECT'.freeze
+          # @deprecated Use `DD_TRACE_PROPAGATION_STYLE_INJECT` instead.
+          ENV_PROPAGATION_STYLE_INJECT_OLD = 'DD_PROPAGATION_STYLE_INJECT'.freeze
+
+          ENV_PROPAGATION_STYLE_EXTRACT = 'DD_TRACE_PROPAGATION_STYLE_EXTRACT'.freeze
+          # @deprecated Use `DD_TRACE_PROPAGATION_STYLE_EXTRACT` instead.
+          ENV_PROPAGATION_STYLE_EXTRACT_OLD = 'DD_PROPAGATION_STYLE_EXTRACT'.freeze
+
+          # A no-op propagator. Compatible with OpenTelemetry's `none` propagator.
+          # @see https://opentelemetry.io/docs/concepts/sdk-configuration/general-sdk-configuration/#get_otel__propagators
+          PROPAGATION_STYLE_NONE = 'none'.freeze
+
+          ENV_X_DATADOG_TAGS_MAX_LENGTH = 'DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH'.freeze
         end
 
         # @public_api
@@ -32,6 +63,12 @@ module Datadog
         module Sampling
           ENV_SAMPLE_RATE = 'DD_TRACE_SAMPLE_RATE'.freeze
           ENV_RATE_LIMIT = 'DD_TRACE_RATE_LIMIT'.freeze
+
+          # @public_api
+          module Span
+            ENV_SPAN_SAMPLING_RULES = 'DD_SPAN_SAMPLING_RULES'.freeze
+            ENV_SPAN_SAMPLING_RULES_FILE = 'DD_SPAN_SAMPLING_RULES_FILE'.freeze
+          end
         end
 
         # @public_api
@@ -41,10 +78,16 @@ module Datadog
 
         # @public_api
         module Transport
-          ENV_DEFAULT_HOST = 'DD_AGENT_HOST'.freeze
           ENV_DEFAULT_PORT = 'DD_TRACE_AGENT_PORT'.freeze
           ENV_DEFAULT_URL = 'DD_TRACE_AGENT_URL'.freeze
         end
+
+        # @public_api
+        module ClientIp
+          ENV_ENABLED = 'DD_TRACE_CLIENT_IP_ENABLED'.freeze
+          ENV_DISABLED = 'DD_TRACE_CLIENT_IP_HEADER_DISABLED'.freeze # TODO: deprecated, remove later
+          ENV_HEADER_NAME = 'DD_TRACE_CLIENT_IP_HEADER'.freeze
+        end
       end
     end
   end