datadog 2.35.0 → 2.36.0

data/lib/datadog/appsec/contrib/rack/gateway/request.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative '../input_peeker'
 require_relative '../../../instrumentation/gateway/argument'
 require_relative '../../../../core/header_collection'
 require_relative '../../../../tracing/client_ip'
@@ -75,14 +76,51 @@ module Datadog
             end
 
             def form_hash
-              # force form data processing
+              # NOTE: Rack populates `rack.request.form_hash` only as a side effect
+              #       of {::Rack::Request#POST}, which reads and parses the body.
               request.POST if request.form_data?
 
-              # usually Hash<String,String> but can be a more complex
-              # Hash<String,String||Array||Hash> when e.g coming from JSON
+              # usually Hash[String, String] but can be a more complex
+              # Hash[String, (String|Array|Hash)] when e.g coming from JSON
               env['rack.request.form_hash']
             end
 
+            # Returns the request body size in bytes using all available methods,
+            # or nil when the size cannot be measured within the limit
+            #
+            # NOTE: The priority of the measurement is the following:
+            #       size if it's known, content-length if provided, and buffering
+            #       to the limit if unknown-length
+            #
+            # WARNING: The buffering path adds overhead for streaming web-servers
+            #          (Rack 3+) when the body length is unknown
+            def body_bytesize(limit)
+              io = request.body
+
+              return 0 unless io
+              return io.size if io.respond_to?(:size)
+
+              # NOTE: {::Rack::Request#content_length} is a plain `CONTENT_LENGTH`
+              #       getter forces no body read
+              content_length = request.content_length
+              return content_length.to_i if content_length
+
+              # NOTE: A parsed-body cache without a raw byte count means the input
+              #       stream was consumed before measurement, consider it unknown-length
+              return if env.key?('rack.request.form_hash')
+
+              InputPeeker.peek_bytesize(env, limit: limit)
+            end
+
+            # Whether a request body can be collected without forcing a parse:
+            # either form data parseable on demand, or a body already parsed upstream
+            #
+            # NOTE: Rack does not parse JSON itself, a body parser middleware such as
+            #       {Rack::JSONBodyParser} populates the form hash read by {#form_hash}
+            def collectable_body?
+              request.form_data? || request.parseable_data? || env.key?('rack.request.form_hash')
+            end
+
             def client_ip
               remote_ip = remote_addr
               header_collection = Datadog::Core::HeaderCollection.from_hash(headers)

data/lib/datadog/appsec/contrib/rack/gateway/watcher.rb

@@ -87,9 +87,26 @@ module Datadog
                 gateway.watch('rack.request.body') do |stack, gateway_request|
                   context = gateway_request.env[AppSec::Ext::CONTEXT_KEY]
 
-                  persistent_data = {
-                    'server.request.body' => gateway_request.form_hash
-                  }
+                  request = gateway_request.request
+                  next stack.call(request) unless gateway_request.collectable_body?
+
+                  # NOTE: A limit of 0 disables request body collection entirely.
+                  limit = Datadog.configuration.appsec.body_parsing_size_limit
+                  next stack.call(request) if limit.zero?
+
+                  persistent_data = {}
+                  byte_length = gateway_request.body_bytesize(limit)
+
+                  if byte_length
+                    persistent_data['server.request.body.byte_length'] = byte_length
+
+                    if byte_length <= limit
+                      body = gateway_request.form_hash
+                      persistent_data['server.request.body'] = body if body
+                    end
+                  end
+
+                  next stack.call(request) if persistent_data.empty?
 
                   result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
 
@@ -119,10 +136,6 @@ module Datadog
                   next stack.call(gateway_request.request) if context.span.nil?
 
                   gateway_request.headers.each do |name, value|
-                    if Ext::COLLECTABLE_REQUEST_HEADERS.include?(name)
-                      context.span["http.request.headers.#{name}"] ||= value
-                    end
-
                     if context.state[:has_identity_event] && Ext::IDENTITY_COLLECTABLE_REQUEST_HEADERS.include?(name)
                       context.span["http.request.headers.#{name}"] ||= value
                     end

data/lib/datadog/appsec/contrib/rack/input_peeker.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+require 'stringio'
+require_relative 'buffered_input'
+
+module Datadog
+  module AppSec
+    module Contrib
+      module Rack
+        # Peeks at `rack.input` without changing what downstream Rack code can read
+        #
+        # @api private
+        module InputPeeker
+          module_function
+
+          # Peeks at `env['rack.input']` and returns the body bytesize when it
+          # can be measured within the given limit
+          #
+          # NOTE: Over-limit bodies cannot be fully measured, so returning `nil`
+          #       is the tradeoff we accept
+          #
+          # WARNING: For forward-only input, replaces `env['rack.input']` with a
+          #          replay stream over the peeked bytes, preserving the rest
+          #          of the original stream
+          #
+          # @param env [Hash] Rack environment
+          # @param limit [Integer] Maximum body bytesize to measure
+          # @return [Integer, nil] `Integer` bytesize when measured within
+          #   `limit`, including `0`; `nil` when input is missing, over the
+          #   limit, or cannot be preserved for downstream reads
+          def peek_bytesize(env, limit:)
+            rack_input = env['rack.input']
+            return unless rack_input
+
+            rewindable = rewind? && rack_input.respond_to?(:rewind)
+
+            # NOTE: Rack 2 requires `rack.input` to be rewindable. Rewind before peeking
+            #       in case an upstream framework already consumed part of the stream
+            return if rewindable && !rewind(rack_input)
+
+            buffer = peek(rack_input, limit)
+            over_limit = buffer.bytesize > limit
+
+            if rewindable
+              # NOTE: If we cannot rewind after peeking, downstream code would observe
+              #       a partially consumed body. Treat it as not safely collectable
+              return unless rewind(rack_input)
+            else
+              env['rack.input'] = if over_limit
+                BufferedInput.new(rack_input, buffer: StringIO.new(buffer))
+              else
+                StringIO.new(buffer)
+              end
+            end
+
+            over_limit ? nil : buffer.bytesize
+          end
+
+          private_class_method def rewind?
+            return @rewind if defined?(@rewind)
+
+            @rewind = ::Gem::Version.new(::Rack.release) < ::Gem::Version.new('3')
+          end
+
+          private_class_method def rewind(io)
+            io.rewind
+            true
+          rescue => e
+            Datadog.logger.debug { "AppSec: Failed to rewind `rack.input`: #{e.class}: #{e.message}" }
+            false
+          end
+
+          private_class_method def peek(io, limit)
+            # NOTE: Read one byte past the limit to distinguish an exact-limit body
+            #       from an over-limit body without reading the whole stream.
+            max = limit + 1
+            buffer = +''.b
+
+            while buffer.bytesize <= limit
+              chunk = io.read(max - buffer.bytesize)
+              break if chunk.nil? || chunk.empty?
+
+              buffer << chunk
+            end
+
+            buffer
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/rails/gateway/request.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative '../../rack/input_peeker'
 require_relative '../../../instrumentation/gateway/argument'
 
 module Datadog
@@ -59,6 +60,38 @@ module Datadog
                 excluded.include?(k)
               end
             end
+
+            # Returns the request body size in bytes using all available methods,
+            # or nil when the size cannot be measured within the limit
+            #
+            # NOTE: The priority of the measurement is the following:
+            #       raw posted data, raw form vars, size if known, raw
+            #       Content-Length, then buffering to the limit if unknown-length
+            def body_bytesize(limit)
+              raw_body = env['RAW_POST_DATA']
+              return raw_body.bytesize if raw_body
+
+              form_vars = env['rack.request.form_vars']
+              return form_vars.bytesize if form_vars
+
+              io = request.body
+              return 0 unless io
+              return io.size if io.respond_to?(:size)
+
+              # NOTE: Read raw `CONTENT_LENGTH` as {ActionDispatch::Request#content_length}
+              #       drains `rack.input` into `RAW_POST_DATA` on chunked Transfer-Encoding
+              content_length = env['CONTENT_LENGTH']
+              return content_length.to_i if content_length
+
+              # NOTE: An already-read body (e.g. late-parsed multipart on Rack 3+) peeks
+              #       as 0, so we skip byte_length but still collect the parsed body
+              begin
+                Rack::InputPeeker.peek_bytesize(env, limit: limit)
+              rescue => e
+                Datadog.logger.debug { "AppSec: Failed to measure Rails request body: #{e.class}: #{e.message}" }
+                nil
+              end
+            end
           end
         end
       end

data/lib/datadog/appsec/contrib/rails/gateway/watcher.rb

@@ -23,12 +23,28 @@ module Datadog
               def watch_request_action(gateway = Instrumentation.gateway)
                 gateway.watch('rails.request.action') do |stack, gateway_request|
                   context = gateway_request.env[AppSec::Ext::CONTEXT_KEY]
+                  limit = Datadog.configuration.appsec.body_parsing_size_limit
 
                   persistent_data = {
-                    'server.request.body' => gateway_request.parsed_body,
                     'server.request.path_params' => gateway_request.route_params
                   }
 
+                  unless limit.zero?
+                    byte_length = gateway_request.body_bytesize(limit)
+
+                    if byte_length
+                      # NOTE: Params may be parsed before this hook, leaving the
+                      #       body stream at EOF. Keep byte_length unset for that
+                      #       case, but still inspect cached params
+                      persistent_data['server.request.body.byte_length'] = byte_length if byte_length.positive?
+
+                      if byte_length <= limit
+                        body = gateway_request.parsed_body
+                        persistent_data['server.request.body'] = body unless body.nil? || body.empty?
+                      end
+                    end
+                  end
+
                   result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
 
                   if result.match?

data/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb

@@ -27,9 +27,26 @@ module Datadog
 
                   context.state[:web_framework] = 'sinatra'
 
-                  persistent_data = {
-                    'server.request.body' => gateway_request.form_hash
-                  }
+                  request = gateway_request.request
+                  next stack.call(request) unless gateway_request.collectable_body?
+
+                  # NOTE: A limit of 0 disables request body collection entirely.
+                  limit = Datadog.configuration.appsec.body_parsing_size_limit
+                  next stack.call(request) if limit.zero?
+
+                  persistent_data = {}
+                  byte_length = gateway_request.body_bytesize(limit)
+
+                  if byte_length
+                    persistent_data['server.request.body.byte_length'] = byte_length
+
+                    if byte_length <= limit
+                      body = gateway_request.form_hash
+                      persistent_data['server.request.body'] = body if body
+                    end
+                  end
+
+                  next stack.call(request) if persistent_data.empty?
 
                   result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
 

data/lib/datadog/appsec/default_header_tags.rb

@@ -10,20 +10,24 @@ module Datadog
     #
     # @api private
     module DefaultHeaderTags
-      WAF_VENDOR_HEADERS_TAGS = %w[
+      # NOTE: Contains additional WAF vendor headers
+      REQUEST_HEADERS_TAGS = %w[
+        accept
+        content-type
+        user-agent
+        akamai-user-risk
         x-amzn-trace-id
-        cloudfront-viewer-ja3-fingerprint
-        cf-ray
         x-cloud-trace-context
         x-appgw-trace-id
         x-sigsci-requestid
         x-sigsci-tags
-        akamai-user-risk
+        cf-ray
+        cloudfront-viewer-ja3-fingerprint
       ].freeze
 
       RESPONSE_HEADERS_TAGS = %w[
-        content-length
         content-type
+        content-length
         content-encoding
         content-language
       ].freeze
@@ -31,7 +35,7 @@ module Datadog
       module_function
 
       def tag_request(span, headers)
-        WAF_VENDOR_HEADERS_TAGS.each do |name|
+        REQUEST_HEADERS_TAGS.each do |name|
           value = headers.get(name)
           span.set_tag("http.request.headers.#{name}", value) if value
         end

data/lib/datadog/core/configuration/components.rb

@@ -194,6 +194,7 @@ module Datadog
           remote&.after_fork
           crashtracker&.update_on_fork
           ProcessDiscovery.after_fork
+          symbol_database&.after_fork!
         end
 
         # Hot-swaps with a new sampler.

data/lib/datadog/core/configuration/settings.rb

@@ -618,8 +618,7 @@ module Datadog
               o.type :bool
               o.env 'DD_PROFILING_SIGHANDLER_SAMPLING_ENABLED'
               o.default do
-                Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('3.2.5') &&
-                  !(RUBY_VERSION.start_with?('3.3.') && Gem::Version.new(RUBY_VERSION) < Gem::Version.new('3.3.4'))
+                RubyVersion.is?('>= 3.2.5') && !RubyVersion.is?('>= 3.3', '< 3.3.4')
               end
             end
 

data/lib/datadog/core/configuration/supported_configurations.rb

@@ -25,6 +25,7 @@ module Datadog
           "DD_APM_TRACING_ENABLED",
           "DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING",
           "DD_APPSEC_AUTO_USER_INSTRUMENTATION_MODE",
+          "DD_APPSEC_BODY_PARSING_SIZE_LIMIT",
           "DD_APPSEC_ENABLED",
           "DD_APPSEC_HTTP_BLOCKED_TEMPLATE_HTML",
           "DD_APPSEC_HTTP_BLOCKED_TEMPLATE_JSON",
@@ -270,6 +271,7 @@ module Datadog
           "DD_TRACE_PRESTO_ENABLED",
           "DD_TRACE_PRESTO_PEER_SERVICE",
           "DD_TRACE_PRESTO_SERVICE_NAME",
+          "DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT",
           "DD_TRACE_PROPAGATION_EXTRACT_FIRST",
           "DD_TRACE_PROPAGATION_STYLE",
           "DD_TRACE_PROPAGATION_STYLE_EXTRACT",

data/lib/datadog/core/remote/component.rb

@@ -144,7 +144,7 @@ module Datadog
                 #   timeout and an integer otherwise
                 # - before Ruby 3.2, ConditionVariable returns itself
                 # so we have to rely on @once having been set
-                if RUBY_VERSION >= '3.2'
+                if RubyVersion.is?('>= 3.2')
                   lifted = @condition.wait(@mutex, timeout)
                 else
                   @condition.wait(@mutex, timeout)

data/lib/datadog/core/telemetry/event/app_started.rb

@@ -146,27 +146,6 @@ module Datadog
               )
             end
 
-            # OpenTelemetry configuration options (using environment variable names)
-            otel_exporter_headers_option = resolve_option(settings, 'opentelemetry.exporter.headers')
-            otel_exporter_headers_option.values_per_precedence.each do |precedence, value|
-              list << conf_value(
-                option_telemetry_name(otel_exporter_headers_option),
-                # Steep: Value is always a hash for opentelemetry.exporter.headers (ensured by o.type :hash)
-                value&.map { |key, header_value| "#{key}=#{header_value}" }&.join(','), # steep:ignore NoMethod
-                precedence
-              )
-            end
-
-            otel_metrics_headers_option = resolve_option(settings, 'opentelemetry.metrics.headers')
-            otel_metrics_headers_option.values_per_precedence.each do |precedence, value|
-              list << conf_value(
-                option_telemetry_name(otel_metrics_headers_option),
-                # Steep: Value is always a hash for opentelemetry.metrics.headers (ensured by o.type :hash)
-                value&.map { |key, header_value| "#{key}=#{header_value}" }&.join(','), # steep:ignore NoMethod
-                precedence
-              )
-            end
-
             # Add some more custom additional payload values here
             if settings.logger.instance
               logger_instance_option = resolve_option(settings, 'logger.instance')

data/lib/datadog/core/utils/at_fork_monkey_patch.rb

@@ -16,7 +16,7 @@ module Datadog
         def self.apply!
           return false unless supported?
 
-          if RUBY_VERSION < '3.1'
+          if RubyVersion.is?('< 3.1')
             [
               ::Process.singleton_class, # Process.fork
               ::Kernel.singleton_class,  # Kernel.fork

data/lib/datadog/core/utils/forking.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative '../../ruby_version'
+
 module Datadog
   module Core
     module Utils
@@ -45,7 +47,7 @@ module Datadog
           # object will cause forking to not be detected in the fork when it should have.
           #
           # This wrapper prevents this by initializing the fork PID when the object is created.
-          if RUBY_VERSION >= '3'
+          if RubyVersion.is?('>= 3')
             def initialize(*args, **kwargs, &block)
               super
               update_fork_pid!

data/lib/datadog/core/utils/spawn_monkey_patch.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative '../../ruby_version'
+
 module Datadog
   module Core
     module Utils
@@ -19,7 +21,7 @@ module Datadog
         # Prepends `Process.spawn` to merge `env_provider` output into the child's environment hash.
         module ProcessSpawnPatch
           # The One and Only Correct Delegation Pattern
-          if RUBY_VERSION >= '3'
+          if RubyVersion.is?('>= 3')
             def spawn(*args, **kwargs) # steep:ignore DifferentMethodParameterKind
               super(*SpawnMonkeyPatch.inject_envs(args), **kwargs)
             end

data/lib/datadog/core.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+# Keep this at the top, this is needed at require-time by some files
+require_relative 'ruby_version'
+
 require_relative 'core/deprecations'
 require_relative 'core/configuration/config_helper'
 require_relative 'core/extensions'

data/lib/datadog/di/base.rb

@@ -10,6 +10,9 @@
 
 require_relative 'code_tracker'
 
+# Needed since this file can be loaded without core
+require_relative '../ruby_version'
+
 module Datadog
   # Namespace for Datadog dynamic instrumentation.
   #
@@ -44,7 +47,7 @@ module Datadog
       # DI code.
       def activate_tracking
         # :script_compiled trace point was added in Ruby 2.6.
-        return unless RUBY_VERSION >= '2.6'
+        return unless RubyVersion.is?('>= 2.6')
 
         begin
           # Activate code tracking by default because line trace points will not work

data/lib/datadog/di/component.rb

@@ -45,7 +45,7 @@ module Datadog
             logger.warn("di: cannot enable dynamic instrumentation: MRI is required, but running on #{RUBY_ENGINE}")
             return false
           end
-          if RUBY_VERSION < '2.6'
+          if RubyVersion.is?('< 2.6')
             logger.warn("di: cannot enable dynamic instrumentation: Ruby 2.6+ is required, but running on #{RUBY_VERSION}")
             return false
           end

data/lib/datadog/error_tracking/collector.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative 'ext'
+require_relative '../ruby_version'
 
 module Datadog
   module ErrorTracking
@@ -52,7 +53,7 @@ module Datadog
         end
       end
 
-      if RUBY_VERSION >= Ext::RUBY_VERSION_WITH_RESCUE_EVENT
+      if RubyVersion.is?(">= #{Ext::RUBY_VERSION_WITH_RESCUE_EVENT}")
         # Starting from ruby3.3, as we are listening to :rescue event,
         # we just want to remove the span event if the error was
         # previously handled

data/lib/datadog/error_tracking/component.rb

@@ -33,7 +33,7 @@ module Datadog
           if RUBY_ENGINE != 'ruby'
             logger.warn("error tracking: cannot enable error tracking: MRI is required, but running on #{RUBY_ENGINE}")
             false
-          elsif RUBY_VERSION < '2.7'
+          elsif RubyVersion.is?('< 2.7')
             logger.warn(
               "error tracking: cannot enable error tracking: Ruby 2.7+ is required, but running
               on #{RUBY_VERSION}"
@@ -63,7 +63,7 @@ module Datadog
         # Before Ruby3.3 the TracePoint listen for :raise events.
         # If an error is not handled, we will delete the according
         # span event in the collector.
-        event = (RUBY_VERSION >= '3.3') ? :rescue : :raise
+        event = RubyVersion.is?('>= 3.3') ? :rescue : :raise
 
         # This TracePoint is in charge of capturing the handled exceptions
         # and of adding the corresponding span events to the collector

data/lib/datadog/kit/tracing/method_tracer.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+# Needed since this file can be loaded without core
+require_relative '../../ruby_version'
+
 module Datadog
   module Kit
     module Tracing
@@ -72,7 +75,7 @@ module Datadog
               raise ArgumentError, 'span name is not a String'
             end
 
-            args = (RUBY_VERSION >= '2.7.') ? '...' : '*args, &block'
+            args = RubyVersion.is?('>= 2.7') ? '...' : '*args, &block'
 
             hook_module = Module.new do
               define_singleton_method(:inspect) do

data/lib/datadog/opentelemetry/sdk/propagator.rb

@@ -43,6 +43,15 @@ module Datadog
           digest = @datadog_propagator.extract(carrier)
           return context unless digest
 
+          # Always call continue_trace! so span links are created in restart mode
+          # (when digest.trace_id is nil) before we attempt to build the OTel SpanContext.
+          trace = Tracing.continue_trace!(digest)
+
+          # In restart mode DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT=restart, the extracted
+          # digest carries no trace_id (fresh start). continue_trace! has already recorded
+          # the span link; return the current context so the caller's span starts a new root.
+          return context unless digest.trace_id
+
           # Converts the {Numeric} Datadog id object to OpenTelemetry's byte array format.
           # 128-bit unsigned, big-endian integer
           trace_id = [digest.trace_id >> 64, digest.trace_id & 0xFFFFFFFFFFFFFFFF].pack('Q>Q>')
@@ -70,9 +79,6 @@ module Datadog
           )
 
           span = ::OpenTelemetry::Trace.non_recording_span(span_context)
-
-          trace = Tracing.continue_trace!(digest)
-
           span.datadog_trace = trace
 
           ::OpenTelemetry::Trace.context_with_span(span, parent_context: context)

data/lib/datadog/opentelemetry/sdk/span_processor.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative 'trace/span'
+require_relative '../../core/utils'
 require_relative '../../tracing/span_link'
 require_relative '../../tracing/span_event'
 require_relative '../../tracing/trace_digest'
@@ -101,12 +102,14 @@ module Datadog
 
           unless span.links.nil?
             datadog_span.links = span.links.map do |link|
+              tracestate = link.span_context.tracestate&.to_s
+
               Datadog::Tracing::SpanLink.new(
                 Datadog::Tracing::TraceDigest.new(
                   trace_id: link.span_context.hex_trace_id.to_i(16),
                   span_id: link.span_context.hex_span_id.to_i(16),
                   trace_sampling_priority: (link.span_context.trace_flags&.sampled? ? 1 : 0),
-                  trace_state: link.span_context.tracestate&.to_s,
+                  trace_state: tracestate && ::Datadog::Core::Utils.utf8_encode(tracestate, placeholder: nil),
                   span_remote: link.span_context.remote?,
                 ),
                 attributes: link.attributes

data/lib/datadog/profiling/collectors/thread_context.rb

@@ -33,6 +33,7 @@ module Datadog
             waiting_for_gvl_threshold_ns: waiting_for_gvl_threshold_ns,
             otel_context_enabled: otel_context_enabled,
             native_filenames_enabled: validate_native_filenames(native_filenames_enabled),
+            overhead_filename: __FILE__,
           )
         end