datadog 2.26.0 → 2.28.0

data/lib/datadog/appsec/utils/http/media_type.rb

@@ -4,14 +4,13 @@ module Datadog
   module AppSec
     module Utils
       module HTTP
-        # Implementation of media type for content negotiation
+        # Implementation of media type for HTTP headers
         #
         # See:
         # - https://www.rfc-editor.org/rfc/rfc7231#section-5.3.1
         # - https://www.rfc-editor.org/rfc/rfc7231#section-5.3.2
         class MediaType
-          class ParseError < ::StandardError
-          end
+          ParseError = Class.new(StandardError) # steep:ignore IncompatibleAssignment
 
           WILDCARD = '*'
 
@@ -19,7 +18,7 @@ module Datadog
           TOKEN_RE = /[-#$%&'*+.^_`|~A-Za-z0-9]+/.freeze
 
           # See: https://www.rfc-editor.org/rfc/rfc7231#section-3.1.1.1
-          PARAMETER_RE = %r{ # rubocop:disable Style/RegexpLiteral
+          PARAMETER_RE = %r{
             (?:
               (?<parameter_name>#{TOKEN_RE})
               =
@@ -46,39 +45,54 @@ module Datadog
 
           attr_reader :type, :subtype, :parameters
 
-          def initialize(media_type)
-            media_type_match = MEDIA_TYPE_RE.match(media_type)
+          def self.json?(media_type)
+            return false if media_type.nil? || media_type.empty?
 
-            raise ParseError, media_type.inspect if media_type_match.nil?
+            match = MEDIA_TYPE_RE.match(media_type)
+            return false if match.nil?
 
-            @type = (media_type_match['type'] || WILDCARD).downcase
-            @subtype = (media_type_match['subtype'] || WILDCARD).downcase
-            @parameters = {}
+            subtype = match['subtype']
+            return false if subtype.nil? || subtype.empty?
 
-            parameters = media_type_match['parameters']
+            subtype.downcase!
+            subtype == 'json' || subtype.end_with?('+json')
+          end
 
-            return if parameters.nil?
+          def initialize(media_type)
+            match = MEDIA_TYPE_RE.match(media_type)
+            raise ParseError, media_type.inspect if match.nil?
 
-            parameters.split(';').map(&:strip).each do |parameter|
-              parameter_match = PARAMETER_RE.match(parameter)
+            @type = match['type'] || WILDCARD
+            @type.downcase!
 
-              next if parameter_match.nil?
+            @subtype = match['subtype'] || WILDCARD
+            @subtype.downcase!
 
-              parameter_name = parameter_match['parameter_name']
-              parameter_value = parameter_match['parameter_value']
+            @parameters = {}
+
+            parameters = match['parameters']
+            return if parameters.nil? || parameters.empty?
 
-              next if parameter_name.nil? || parameter_value.nil?
+            parameters.scan(PARAMETER_RE) do |name, unquoted_value, quoted_value|
+              # NOTE: Order of unquoted_value and quoted_value does not matter,
+              #       as they are mutually exclusive by the regex.
+              # @type var value: ::String?
+              value = unquoted_value || quoted_value
+              next if name.nil? || value.nil?
 
-              @parameters[parameter_name.downcase] = parameter_value.downcase
+              # See https://github.com/soutaro/steep/issues/2051
+              name.downcase! # steep:ignore NoMethod
+              value.downcase!
+
+              # See https://github.com/soutaro/steep/issues/2051
+              @parameters[name] = value # steep:ignore ArgumentTypeMismatch
             end
           end
 
           def to_s
-            s = +"#{@type}/#{@subtype}"
-
-            s << ';' << @parameters.map { |k, v| "#{k}=#{v}" }.join(';') if @parameters.count > 0
+            return "#{@type}/#{@subtype}" if @parameters.empty?
 
-            s
+            "#{@type}/#{@subtype};#{@parameters.map { |k, v| "#{k}=#{v}" }.join(";")}"
           end
         end
       end

data/lib/datadog/appsec.rb

@@ -22,8 +22,14 @@ module Datadog
         Datadog::AppSec::Context.active
       end
 
+      # NOTE:  This is a temporary workaround for type checking.
+      #
+      #        We want to move from possible nil-component to the disabled-component
+      #        on an initialization error. Technically, telemetry will be never
+      #        used if AppSec was not able to initialize, so it's safe to assume
+      #        that telemetry will never be used and will be nil at the same time.
       def telemetry
-        components.appsec&.telemetry
+        components.appsec&.telemetry || components.telemetry
       end
 
       def security_engine

data/lib/datadog/core/configuration/settings.rb

@@ -436,6 +436,23 @@ module Datadog
               o.default true
             end
 
+            # The profiler gathers data by sending `SIGPROF` unix signals to Ruby application threads.
+            #
+            # When using `Kernel#exec` on Linux, it can happen that a signal sent before calling `exec` arrives after
+            # the new process is running, causing it to fail with the `Profiling timer expired` error message.
+            # To avoid this, the profiler installs a monkey patch on `Kernel#exec` to stop profiling before actually
+            # calling `exec`.
+            # This monkey patch is available for Ruby 2.7+; let us know if you need it on earlier Rubies.
+            # For more details see https://github.com/DataDog/dd-trace-rb/issues/5101 .
+            #
+            # @default `DD_PROFILING_SHUTDOWN_ON_EXEC_ENABLED` environment variable as a boolean,
+            # otherwise `true`
+            option :shutdown_on_exec_enabled do |o|
+              o.env 'DD_PROFILING_SHUTDOWN_ON_EXEC_ENABLED'
+              o.type :bool
+              o.default true
+            end
+
             # Configures how much wall-time overhead the profiler targets. The profiler will dynamically adjust the
             # interval between samples it takes so as to try and maintain the property that it spends no longer than
             # this amount of wall-clock time profiling. For example, with the default value of 2%, the profiler will

data/lib/datadog/core/configuration/supported_configurations.rb

@@ -48,6 +48,7 @@ module Datadog
           "DD_DYNAMIC_INSTRUMENTATION_PROBE_FILE",
           "DD_DYNAMIC_INSTRUMENTATION_REDACTED_IDENTIFIERS",
           "DD_DYNAMIC_INSTRUMENTATION_REDACTED_TYPES",
+          "DD_DYNAMIC_INSTRUMENTATION_REDACTION_EXCLUDED_IDENTIFIERS",
           "DD_ENV",
           "DD_ERROR_TRACKING_HANDLED_ERRORS",
           "DD_ERROR_TRACKING_HANDLED_ERRORS_INCLUDE",
@@ -81,6 +82,7 @@ module Datadog
           "DD_PROFILING_NO_SIGNALS_WORKAROUND_ENABLED",
           "DD_PROFILING_OVERHEAD_TARGET_PERCENTAGE",
           "DD_PROFILING_PREVIEW_OTEL_CONTEXT_ENABLED",
+          "DD_PROFILING_SHUTDOWN_ON_EXEC_ENABLED",
           "DD_PROFILING_SIGHANDLER_SAMPLING_ENABLED",
           "DD_PROFILING_SKIP_MYSQL2_CHECK",
           "DD_PROFILING_TIMELINE_ENABLED",

data/lib/datadog/core/crashtracking/tag_builder.rb

@@ -2,6 +2,7 @@
 
 require_relative '../tag_builder'
 require_relative '../utils'
+require_relative '../environment/process'
 
 module Datadog
   module Core
@@ -13,6 +14,11 @@ module Datadog
             'is_crash' => 'true',
           )
 
+          if settings.experimental_propagate_process_tags_enabled
+            process_tags = Environment::Process.serialized
+            hash['process_tags'] = process_tags unless process_tags.empty?
+          end
+
           Utils.encode_tags(hash)
         end
       end

data/lib/datadog/core/knuth_sampler.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    # Deterministic sampler using Knuth multiplicative hash algorithm.
+    #
+    # This sampler provides consistent sampling decisions based on an input value,
+    # ensuring the same input always produces the same sampling decision for a given rate.
+    #
+    # The algorithm multiplies the input by a large prime (Knuth factor), takes modulo
+    # to constrain to a fixed range, and compares against a threshold derived from the sample rate.
+    #
+    # @api private
+    # @see https://en.wikipedia.org/wiki/Hash_function#Multiplicative_hashing
+    class KnuthSampler
+      # Maximum unsigned 64-bit integer for uniform distribution across 64-bit input space.
+      UINT64_MAX = (1 << 64) - 1
+      UINT64_MODULO = 1 << 64
+
+      # Golden ratio constant for optimal distribution.
+      # @see https://en.wikipedia.org/wiki/Hash_function#Fibonacci_hashing
+      DEFAULT_KNUTH_FACTOR = 11400714819323198485
+
+      attr_reader :rate
+
+      # @param rate [Float] Sampling rate between +0.0+ and +1.0+ (inclusive).
+      #   +0.0+ means no samples are kept; +1.0+ means all samples are kept.
+      #   Invalid values fall back to +1.0+ (sample everything).
+      # @param knuth_factor [Integer] Multiplicative constant for hashing.
+      #   Different factors produce different sampling distributions.
+      def initialize(rate = 1.0, knuth_factor: DEFAULT_KNUTH_FACTOR)
+        @knuth_factor = knuth_factor
+
+        rate = rate.to_f
+        unless rate >= 0.0 && rate <= 1.0
+          Datadog.logger.warn("Sample rate #{rate} is not between 0.0 and 1.0, falling back to 1.0")
+          rate = 1.0
+        end
+
+        @rate = rate
+        @threshold = @rate * UINT64_MAX
+      end
+
+      # Determines if the given input should be sampled.
+      #
+      # This method is deterministic: the same input value always produces
+      # the same result for a given sample rate and configuration.
+      #
+      # @param input [Integer] Value to determine sampling decision.
+      #   Typically a trace ID or incrementing counter.
+      # @return [Boolean] +true+ if input should be sampled, +false+ otherwise
+      def sample?(input)
+        ((input * @knuth_factor) % UINT64_MODULO) <= @threshold
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/logger.rb

@@ -14,10 +14,12 @@ module Datadog
       #   read: lib/datadog/core/telemetry/logging.rb
       module Logger
         class << self
+          # (see Datadog::Core::Telemetry::Logging#report)
           def report(exception, level: :error, description: nil)
             instance&.report(exception, level: level, description: description)
           end
 
+          # (see Datadog::Core::Telemetry::Logging#error)
           def error(description)
             instance&.error(description)
           end

data/lib/datadog/core/telemetry/logging.rb

@@ -45,11 +45,20 @@ module Datadog
           end
         end
 
+        # @param exception [Exception] The exception to report
+        # @param level [Symbol] The log level (:error, :warn, :info, :debug)
+        # @param description [String, nil] A low cardinality description, without dynamic data
         def report(exception, level: :error, description: nil)
           # Anonymous exceptions to be logged as <Class:0x00007f8b1c0b3b40>
-          message = +"#{exception.class.name || exception.class.inspect}" # standard:disable Style/RedundantInterpolation
+          message = +(exception.class.name || exception.class.inspect)
 
-          message << ": #{description}" if description
+          telemetry_message = message_for_telemetry(exception)
+
+          if description || telemetry_message
+            message << ':'
+            message << " #{description}" if description
+            message << " (#{telemetry_message})" if telemetry_message
+          end
 
           event = Event::Log.new(
             message: message,
@@ -65,6 +74,15 @@ module Datadog
 
           log!(event)
         end
+
+        private
+
+        # A constant string representing the exception
+        def message_for_telemetry(exception)
+          return unless exception.instance_variable_defined?(:@telemetry_message)
+
+          exception.instance_variable_get(:@telemetry_message)
+        end
       end
     end
   end

data/lib/datadog/di/configuration/settings.rb

@@ -45,6 +45,28 @@ module Datadog
                 o.default []
               end
 
+              # An array of variable and key names to exclude from the
+              # built-in redaction list.
+              #
+              # This allows users to capture values of variables that would
+              # otherwise be redacted by the default identifier list.
+              # For example, if an application has a "session" variable
+              # that does not contain sensitive data, "session" can be added
+              # to this list to exclude it from redaction.
+              #
+              # The names will be normalized the same way as redacted_identifiers,
+              # by removing the following symbols: _, -, @, $, and then matched
+              # against the complete variable or key name while ignoring the case.
+              option :redaction_excluded_identifiers do |o|
+                o.env "DD_DYNAMIC_INSTRUMENTATION_REDACTION_EXCLUDED_IDENTIFIERS"
+                o.env_parser do |value|
+                  value&.split(",")&.map(&:strip)
+                end
+
+                o.type :array
+                o.default []
+              end
+
               # An array of class names, values of which will be redacted from
               # dynamic instrumentation snapshots. Example: FooClass.
               # If a name is suffixed by '*', it becomes a wildcard and

data/lib/datadog/di/redactor.rb

@@ -13,6 +13,10 @@ module Datadog
     # redaction. Additional names can be provided by the user via the
     # settings.dynamic_instrumentation.redacted_identifiers setting or
     # the DD_DYNAMIC_INSTRUMENTATION_REDACTED_IDENTIFIERS environment
+    # variable. Users can also exclude specific identifiers from the default
+    # redaction list via the
+    # settings.dynamic_instrumentation.redaction_excluded_identifiers setting or
+    # the DD_DYNAMIC_INSTRUMENTATION_REDACTION_EXCLUDED_IDENTIFIERS environment
     # variable. Currently no class names are subject to redaction by default;
     # class names can be provided via the
     # settings.dynamic_instrumentation.redacted_type_names setting or
@@ -61,7 +65,10 @@ module Datadog
           names.map! do |name|
             normalize(name)
           end
-          Set.new(names)
+          excluded = settings.dynamic_instrumentation.redaction_excluded_identifiers.map do |name|
+            normalize(name)
+          end
+          Set.new(names) - Set.new(excluded)
         end
       end
 

data/lib/datadog/profiling/component.rb

@@ -82,6 +82,10 @@ module Datadog
           Datadog::Profiling::Ext::DirMonkeyPatches.apply!
         end
 
+        if can_apply_exec_monkey_patch?(settings)
+          Datadog::Profiling::Ext::ExecMonkeyPatch.apply!
+        end
+
         [profiler, {profiling_enabled: true}]
       end
 
@@ -434,6 +438,15 @@ module Datadog
         settings.profiling.advanced.dir_interruption_workaround_enabled
       end
 
+      private_class_method def self.can_apply_exec_monkey_patch?(settings)
+        return false if RUBY_VERSION < "2.7"
+
+        # This file is 2.7+ only so we only require it here once we've checked the Ruby version
+        require "datadog/profiling/ext/exec_monkey_patch"
+
+        settings.profiling.advanced.shutdown_on_exec_enabled
+      end
+
       private_class_method def self.enable_gvl_profiling?(settings, logger)
         return false if RUBY_VERSION < "3.2"
 

data/lib/datadog/profiling/exporter.rb

@@ -70,6 +70,9 @@ module Datadog
 
         uncompressed_code_provenance = code_provenance_collector.refresh.generate_json if code_provenance_collector
 
+        process_tags = Datadog.configuration.experimental_propagate_process_tags_enabled ?
+          Core::Environment::Process.serialized : ''
+
         Flush.new(
           start: start,
           finish: finish,
@@ -80,6 +83,7 @@ module Datadog
             settings: Datadog.configuration,
             profile_seq: sequence_tracker.get_next,
           ).to_a,
+          process_tags: process_tags,
           internal_metadata: internal_metadata.merge(
             {
               worker_stats: worker_stats,

data/lib/datadog/profiling/ext/exec_monkey_patch.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Profiling
+    module Ext
+      # The profiler gathers data by sending `SIGPROF` unix signals to Ruby application threads.
+      #
+      # When using `Kernel#exec` on Linux, it can happen that a signal sent before calling `exec` arrives after
+      # the new process is running, causing it to fail with the `Profiling timer expired` error message.
+      # To avoid this, the profiler installs a monkey patch on `Kernel#exec` to stop profiling before actually
+      # calling `exec`.
+      # This monkey patch is available for Ruby 2.7+; let us know if you need it on earlier Rubies.
+      # For more details see https://github.com/DataDog/dd-trace-rb/issues/5101 .
+      module ExecMonkeyPatch
+        def self.apply!
+          ::Object.prepend(ObjectMonkeyPatch)
+
+          true
+        end
+
+        module ObjectMonkeyPatch
+          private
+
+          def exec(...)
+            Datadog.send(:components, allow_initialization: false)&.profiler&.shutdown!(report_last_profile: false)
+            super
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/profiling/flush.rb

@@ -13,6 +13,7 @@ module Datadog
         :code_provenance_file_name,
         :code_provenance_data,
         :tags_as_array,
+        :process_tags,
         :internal_metadata_json,
         :info_json
 
@@ -23,6 +24,7 @@ module Datadog
         code_provenance_file_name:,
         code_provenance_data:,
         tags_as_array:,
+        process_tags:,
         internal_metadata:,
         info_json:
       )
@@ -32,6 +34,7 @@ module Datadog
         @code_provenance_file_name = code_provenance_file_name
         @code_provenance_data = code_provenance_data
         @tags_as_array = tags_as_array
+        @process_tags = process_tags
         @internal_metadata_json = JSON.generate(internal_metadata)
         @info_json = info_json
       end

data/lib/datadog/profiling/profiler.rb

@@ -31,10 +31,11 @@ module Datadog
         scheduler.start(on_failure_proc: proc { component_failed(:scheduler) })
       end
 
-      def shutdown!
+      def shutdown!(report_last_profile: true)
         Datadog.logger.debug("Shutting down profiler")
 
         stop_worker
+        scheduler.disable_reporting unless report_last_profile
         stop_scheduler
       end
 
@@ -57,11 +58,8 @@ module Datadog
         Datadog::Core::Telemetry::Logger
           .error("Detected issue with profiler (#{failed_component} component), stopping profiling")
 
-        # We explicitly not stop the crash tracker in this situation, under the assumption that, if a component failed,
-        # we're operating in a degraded state and crash tracking may still be helpful.
-
         if failed_component == :worker
-          scheduler.mark_profiler_failed
+          scheduler.disable_reporting
           stop_scheduler
         elsif failed_component == :scheduler
           stop_worker

data/lib/datadog/profiling/scheduler.rb

@@ -24,7 +24,7 @@ module Datadog
       attr_reader \
         :exporter,
         :transport,
-        :profiler_failed
+        :reporting_disabled
 
       public
 
@@ -36,7 +36,7 @@ module Datadog
       )
         @exporter = exporter
         @transport = transport
-        @profiler_failed = false
+        @reporting_disabled = false
         @stop_requested = false
 
         # Workers::Async::Thread settings
@@ -83,14 +83,15 @@ module Datadog
         true
       end
 
-      # This is called by the Profiler class whenever an issue happened in the profiler. This makes sure that even
-      # if there is data to be flushed, we don't try to flush it.
-      def mark_profiler_failed
-        @profiler_failed = true
+      # Called by the Profiler when it wants to prevent any further reporting,
+      # even if there is data to be flushed.
+      # Used when the profiler failed or we're otherwise in a hurry to shut down.
+      def disable_reporting
+        @reporting_disabled = true
       end
 
       def work_pending?
-        !profiler_failed && exporter.can_flush? && (run_loop? || !stop_requested?)
+        !reporting_disabled && exporter.can_flush? && (run_loop? || !stop_requested?)
       end
 
       def reset_after_fork

data/lib/datadog/profiling/tag_builder.rb

@@ -50,6 +50,7 @@ module Datadog
           FORM_FIELD_TAG_PROFILER_VERSION => profiler_version,
           'profile_seq' => profile_seq.to_s,
         )
+
         user_tag_keys = settings.tags.keys
         hash.keep_if { |tag| user_tag_keys.include?(tag) || ALLOWED_TAGS.include?(tag) }
         Core::Utils.encode_tags(hash)

data/lib/datadog/tracing/sampling/rate_sampler.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require_relative 'sampler'
-require_relative '../utils'
+require_relative '../../core/knuth_sampler'
 
 module Datadog
   module Tracing
@@ -9,46 +9,35 @@ module Datadog
       # {Datadog::Tracing::Sampling::RateSampler} is based on a sample rate.
       class RateSampler < Sampler
         KNUTH_FACTOR = 1111111111111111111
-        UINT64_MODULO = (1 << 64)
 
         # Initialize a {Datadog::Tracing::Sampling::RateSampler}.
         # This sampler keeps a random subset of the traces. Its main purpose is to
         # reduce the instrumentation footprint.
         #
         # @param sample_rate [Numeric] the sample rate between 0.0 and 1.0, inclusive.
-        #   0.0 means that no trace will be sampled; 1.0 means that all traces will be  sampled.
+        #   0.0 means that no trace will be sampled; 1.0 means that all traces will be sampled.
         def initialize(sample_rate = 1.0, decision: nil)
           super()
-
-          unless sample_rate >= 0.0 && sample_rate <= 1.0
-            Datadog.logger.warn('sample rate is not between 0 and 1, falling back to 1')
-            sample_rate = 1.0
-          end
-
-          self.sample_rate = sample_rate
-
+          @sampler = Core::KnuthSampler.new(sample_rate, knuth_factor: KNUTH_FACTOR)
           @decision = decision
         end
 
         def sample_rate(*_)
-          @sample_rate
+          @sampler.rate
         end
 
         def sample_rate=(sample_rate)
-          @sample_rate = sample_rate
-          @sampling_id_threshold = sample_rate * Tracing::Utils::EXTERNAL_MAX_ID
+          @sampler = Core::KnuthSampler.new(sample_rate, knuth_factor: KNUTH_FACTOR)
         end
 
         def sample?(trace)
-          ((trace.id * KNUTH_FACTOR) % UINT64_MODULO) <= @sampling_id_threshold
+          @sampler.sample?(trace.id)
         end
 
         def sample!(trace)
-          sampled = sample?(trace)
-
-          return false unless sampled
+          return false unless sample?(trace)
 
-          trace.sample_rate = @sample_rate
+          trace.sample_rate = sample_rate
           trace.set_tag(Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER, @decision) if @decision
 
           true

data/lib/datadog/version.rb

@@ -3,7 +3,7 @@
 module Datadog
   module VERSION
     MAJOR = 2
-    MINOR = 26
+    MINOR = 28
     PATCH = 0
     PRE = nil
     BUILD = nil