datadog 2.29.0 → 2.31.0

data/lib/datadog/appsec/contrib/rack/gateway/watcher.rb

@@ -24,7 +24,7 @@ module Datadog
               end
 
               def watch_request(gateway = Instrumentation.gateway)
-                gateway.watch('rack.request', :appsec) do |stack, gateway_request|
+                gateway.watch('rack.request') do |stack, gateway_request|
                   context = gateway_request.env[AppSec::Ext::CONTEXT_KEY]
 
                   persistent_data = {
@@ -57,7 +57,7 @@ module Datadog
               end
 
               def watch_response(gateway = Instrumentation.gateway)
-                gateway.watch('rack.response', :appsec) do |stack, gateway_response|
+                gateway.watch('rack.response') do |stack, gateway_response|
                   context = gateway_response.context
 
                   persistent_data = {
@@ -84,7 +84,7 @@ module Datadog
               end
 
               def watch_request_body(gateway = Instrumentation.gateway)
-                gateway.watch('rack.request.body', :appsec) do |stack, gateway_request|
+                gateway.watch('rack.request.body') do |stack, gateway_request|
                   context = gateway_request.env[AppSec::Ext::CONTEXT_KEY]
 
                   persistent_data = {
@@ -113,20 +113,19 @@ module Datadog
               #       somewhere closer to identity related monitor.
               # WARNING: The Gateway is a subject of refactoring
               def watch_request_finish(gateway = Instrumentation.gateway)
-                gateway.watch('rack.request.finish', :appsec) do |stack, gateway_request|
+                gateway.watch('rack.request.finish') do |stack, gateway_request|
                   context = gateway_request.env[AppSec::Ext::CONTEXT_KEY]
 
-                  if context.span.nil? || !gateway.pushed?('appsec.events.user_lifecycle')
-                    next stack.call(gateway_request.request)
-                  end
+                  next stack.call(gateway_request.request) if context.span.nil?
 
                   gateway_request.headers.each do |name, value|
-                    if !Ext::COLLECTABLE_REQUEST_HEADERS.include?(name) &&
-                        !Ext::IDENTITY_COLLECTABLE_REQUEST_HEADERS.include?(name)
-                      next
+                    if Ext::COLLECTABLE_REQUEST_HEADERS.include?(name)
+                      context.span["http.request.headers.#{name}"] ||= value
                     end
 
-                    context.span["http.request.headers.#{name}"] ||= value
+                    if context.state[:has_identity_event] && Ext::IDENTITY_COLLECTABLE_REQUEST_HEADERS.include?(name)
+                      context.span["http.request.headers.#{name}"] ||= value
+                    end
                   end
 
                   stack.call(gateway_request.request)

data/lib/datadog/appsec/contrib/rack/integration.rb

@@ -27,7 +27,7 @@ module Datadog
           end
 
           def self.compatible?
-            super && version >= MINIMUM_VERSION
+            !!(super && (version&.>= MINIMUM_VERSION))
           end
 
           def self.auto_instrument?

data/lib/datadog/appsec/contrib/rack/request_middleware.rb

@@ -2,6 +2,7 @@
 
 require 'json'
 
+require_relative 'response_body'
 require_relative 'gateway/request'
 require_relative 'gateway/response'
 
@@ -18,6 +19,13 @@ module Datadog
   module AppSec
     module Contrib
       module Rack
+        RESPONSE_HEADERS_TAGS = %w[
+          content-length
+          content-type
+          content-encoding
+          content-language
+        ].freeze
+
         WAF_VENDOR_HEADERS_TAGS = %w[
           X-Amzn-Trace-Id
           Cloudfront-Viewer-Ja3-Fingerprint
@@ -110,8 +118,9 @@ module Datadog
               ctx.extract_schema!
             end
 
-            AppSec::Event.record(ctx, request: gateway_request, response: gateway_response)
+            AppSec::Event.record(ctx, request: gateway_request)
 
+            add_response_tags(ctx, tmp_response)
             http_response
           ensure
             if ctx
@@ -180,7 +189,6 @@ module Datadog
           # standard:disable Metrics/MethodLength
           def add_request_tags(context, env)
             span = context.span
-
             return unless span
 
             # Always add WAF vendors headers
@@ -202,6 +210,21 @@ module Datadog
           end
           # standard:enable Metrics/MethodLength
 
+          def add_response_tags(context, response)
+            span = context.span
+            return unless span
+
+            RESPONSE_HEADERS_TAGS.each do |name|
+              value = response.headers[name]
+              span.set_tag("http.response.headers.#{name}", value.to_s) if value
+            end
+
+            unless response.headers.key?('content-length')
+              length = ResponseBody.content_length(response.body)
+              span.set_tag('http.response.headers.content-length', length.to_s) if length
+            end
+          end
+
           def oneshot_tags_sent?
             @oneshot_tags_sent
           end

data/lib/datadog/appsec/contrib/rack/response_body.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Contrib
+      module Rack
+        module ResponseBody
+          # NOTE: We compute content length only for fixed-size response bodies,
+          #       ignoring streaming bodies to avoid buffering.
+          #
+          #       On Rack 3.x, `body.to_ary` on a BodyProxy triggers `close` on all
+          #       nested BodyProxy layers. This is safe because web servers, like Puma
+          #       handles already-closed bodies (its own `to_ary` becomes a no-op).
+          #
+          # @see Puma::Response#prepare_response
+          # @see https://github.com/puma/puma/blob/b1271222cbf21868f3fb64154caa4d03936a7b9e/lib/puma/response.rb#L165-L168
+          def self.content_length(body)
+            return unless body.respond_to?(:to_ary)
+
+            # NOTE: When `to_ary` exists but returns `nil`, the body will be
+            #       transferred in chunks and we can't compute content length
+            #       without buffering it.
+            body_ary = body.to_ary
+            return unless body_ary.is_a?(Array)
+
+            body_ary.sum { |part| part.is_a?(String) ? part.bytesize : 0 }
+          rescue => e
+            AppSec.telemetry.report(e, description: 'AppSec: Failed to compute body content length')
+
+            nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/rails/gateway/watcher.rb

@@ -21,7 +21,7 @@ module Datadog
               end
 
               def watch_request_action(gateway = Instrumentation.gateway)
-                gateway.watch('rails.request.action', :appsec) do |stack, gateway_request|
+                gateway.watch('rails.request.action') do |stack, gateway_request|
                   context = gateway_request.env[AppSec::Ext::CONTEXT_KEY]
 
                   persistent_data = {
@@ -47,7 +47,7 @@ module Datadog
               end
 
               def watch_response_body_json(gateway = Instrumentation.gateway)
-                gateway.watch('rails.response.body.json', :appsec) do |stack, container|
+                gateway.watch('rails.response.body.json') do |stack, container|
                   context = container.context
 
                   persistent_data = {

data/lib/datadog/appsec/contrib/rails/integration.rb

@@ -26,7 +26,7 @@ module Datadog
           end
 
           def self.compatible?
-            super && version >= MINIMUM_VERSION
+            !!(super && (version&.>= MINIMUM_VERSION))
           end
 
           def self.auto_instrument?

data/lib/datadog/appsec/contrib/rails/patcher.rb

@@ -138,7 +138,7 @@ module Datadog
               end
             rescue => e
               error_message = 'Failed to get application routes'
-              Datadog.logger.error("#{error_message}, #{e.class}: #{e.message}")
+              Datadog.logger.error("#{error_message}, #{e.class}: #{e}")
               AppSec.telemetry.report(e, description: error_message)
             end
           end
@@ -148,7 +148,7 @@ module Datadog
               Datadog::AppSec::Contrib::Rails::Patcher.report_routes_via_telemetry(::Rails.application.routes.routes)
             rescue => e
               error_message = 'Failed to get application routes'
-              Datadog.logger.error("#{error_message}, #{e.class}: #{e.message}")
+              Datadog.logger.error("#{error_message}, #{e.class}: #{e}")
               AppSec.telemetry.report(e, description: error_message)
             end
           end

data/lib/datadog/appsec/contrib/rest_client/patcher.rb

@@ -20,6 +20,8 @@ module Datadog
             require_relative 'request_ssrf_detection_patch'
 
             ::RestClient::Request.prepend(RequestSSRFDetectionPatch)
+
+            Patcher.instance_variable_set(:@patched, true)
           end
         end
       end

data/lib/datadog/appsec/contrib/rest_client/request_ssrf_detection_patch.rb

@@ -50,12 +50,12 @@ module Datadog
               response = super
             rescue ::RestClient::Exception => e
               response = e.response
-              process_response(response, sample_body: sample_body) if response
+              process_response(response, sample_body: sample_body) if response.is_a?(::RestClient::AbstractResponse)
 
               raise
             end
 
-            process_response(response, sample_body: sample_body)
+            process_response(response, sample_body: sample_body) if response.is_a?(::RestClient::AbstractResponse)
             response
           end
 

data/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb

@@ -22,7 +22,7 @@ module Datadog
               end
 
               def watch_request_dispatch(gateway = Instrumentation.gateway)
-                gateway.watch('sinatra.request.dispatch', :appsec) do |stack, gateway_request|
+                gateway.watch('sinatra.request.dispatch') do |stack, gateway_request|
                   context = gateway_request.env[AppSec::Ext::CONTEXT_KEY] # : Context
 
                   context.state[:web_framework] = 'sinatra'
@@ -51,7 +51,7 @@ module Datadog
               end
 
               def watch_request_routed(gateway = Instrumentation.gateway)
-                gateway.watch('sinatra.request.routed', :appsec) do |stack, args|
+                gateway.watch('sinatra.request.routed') do |stack, args|
                   gateway_request, gateway_route_params = args # : [Gateway::Request, Gateway::RouteParams]
                   context = gateway_request.env[AppSec::Ext::CONTEXT_KEY] # : Context
 
@@ -77,7 +77,7 @@ module Datadog
               end
 
               def watch_response_body_json(gateway = Instrumentation.gateway)
-                gateway.watch('sinatra.response.body.json', :appsec) do |stack, container|
+                gateway.watch('sinatra.response.body.json') do |stack, container|
                   context = container.context # : Context
 
                   persistent_data = {

data/lib/datadog/appsec/event.rb

@@ -32,13 +32,6 @@ module Datadog
         accept-language
       ].freeze
 
-      ALLOWED_RESPONSE_HEADERS = %w[
-        content-length
-        content-type
-        content-encoding
-        content-language
-      ].freeze
-
       class << self
         def tag(context, waf_result)
           return if context.span.nil?
@@ -50,7 +43,7 @@ module Datadog
           context.span.set_tag('appsec.event', 'true')
         end
 
-        def record(context, request: nil, response: nil)
+        def record(context, request: nil)
           return if context.events.empty? || context.span.nil?
 
           Datadog::AppSec::RateLimiter.thread_local.limit do
@@ -66,7 +59,6 @@ module Datadog
 
                 context.span['_dd.origin'] = 'appsec'
                 context.span.set_tags(request_tags(request)) if request
-                context.span.set_tags(response_tags(response)) if response
               end
 
               context.span.set_tags(waf_tags(event_group))
@@ -90,14 +82,6 @@ module Datadog
           end
         end
 
-        def response_tags(response)
-          response.headers.each_with_object({}) do |(name, value), memo|
-            next unless ALLOWED_RESPONSE_HEADERS.include?(name)
-
-            memo["http.response.headers.#{name}"] = value
-          end
-        end
-
         def waf_tags(security_events)
           triggers = []
 

data/lib/datadog/appsec/instrumentation/gateway/middleware.rb

@@ -7,10 +7,9 @@ module Datadog
         # NOTE: This class extracted as-is and will be deprecated
         # Instrumentation gateway middleware
         class Middleware
-          attr_reader :key, :block
+          attr_reader :block
 
-          def initialize(key, &block)
-            @key = key
+          def initialize(&block)
             @block = block
           end
 

data/lib/datadog/appsec/instrumentation/gateway.rb

@@ -10,18 +10,9 @@ module Datadog
       class Gateway
         def initialize
           @middlewares = Hash.new { |h, k| h[k] = [] }
-          @pushed_events = {}
         end
 
-        # NOTE: Be careful with pushed names because every pushed event name
-        #       is recorded in order to provide an ability to any subscriber
-        #       to check wether an arbitrary event had happened.
-        #
-        # WARNING: If we start pushing generated names we should consider
-        #          limiting the storage of pushed names.
         def push(name, env, &block)
-          @pushed_events[name] = true
-
           block ||= -> {}
           middlewares_for_name = @middlewares[name]
 
@@ -41,12 +32,8 @@ module Datadog
           stack.call(env)
         end
 
-        def watch(name, key, &block)
-          @middlewares[name] << Middleware.new(key, &block) unless @middlewares[name].any? { |m| m.key == key }
-        end
-
-        def pushed?(name)
-          @pushed_events.key?(name)
+        def watch(name, &block)
+          @middlewares[name] << Middleware.new(&block)
         end
       end
 

data/lib/datadog/appsec/monitor/gateway/watcher.rb

@@ -24,8 +24,9 @@ module Datadog
             end
 
             def watch_user_id(gateway = Instrumentation.gateway)
-              gateway.watch('identity.set_user', :appsec) do |stack, user|
+              gateway.watch('identity.set_user') do |stack, user|
                 context = AppSec.active_context
+                context.state[:has_identity_event] = true
 
                 if user.id.nil? && user.login.nil? && user.session_id.nil?
                   Datadog.logger.debug { 'AppSec: skipping WAF check because no user information was provided' }
@@ -55,8 +56,9 @@ module Datadog
             end
 
             def watch_user_login(gateway = Instrumentation.gateway)
-              gateway.watch('appsec.events.user_lifecycle', :appsec) do |stack, kind|
+              gateway.watch('appsec.events.user_lifecycle') do |stack, kind|
                 context = AppSec.active_context
+                context.state[:has_identity_event] = true
 
                 next stack.call(kind) unless WATCHED_LOGIN_EVENTS.include?(kind)
 

data/lib/datadog/appsec/utils/http/media_type.rb

@@ -68,8 +68,7 @@ module Datadog
                 name.downcase! # steep:ignore NoMethod
                 value.downcase!
 
-                # See https://github.com/soutaro/steep/issues/2051
-                parameters[name] = value # steep:ignore ArgumentTypeMismatch
+                parameters[name] = value
               end
             end
 

data/lib/datadog/appsec/utils/http/url_encoded.rb

@@ -30,9 +30,9 @@ module Datadog
               #
               # @type var key: ::String
               # @type var value: ::String
-              key, value = pair.split('=', 2).map! do |value| #: ::String
+              key, value = pair.split('=', 2).map! do |value|
                 CGI.unescape(value)
-              end
+              end #: [::String, ::String]
 
               if (stored = memo[key])
                 if stored.is_a?(Array)

data/lib/datadog/appsec.rb

@@ -11,25 +11,21 @@ module Datadog
   module AppSec
     class << self
       def enabled?
-        Datadog.configuration.appsec.enabled
+        !!components.appsec
       end
 
       def rasp_enabled?
-        Datadog.configuration.appsec.rasp_enabled
+        # TODO this should take rasp_enabled flag from the settings in
+        # the appsec component rather than reading global configuration.
+        enabled? && Datadog.configuration.appsec.rasp_enabled
       end
 
       def active_context
         Datadog::AppSec::Context.active
       end
 
-      # NOTE:  This is a temporary workaround for type checking.
-      #
-      #        We want to move from possible nil-component to the disabled-component
-      #        on an initialization error. Technically, telemetry will be never
-      #        used if AppSec was not able to initialize, so it's safe to assume
-      #        that telemetry will never be used and will be nil at the same time.
       def telemetry
-        components.appsec&.telemetry || components.telemetry
+        components.telemetry
       end
 
       def security_engine

data/lib/datadog/core/configuration/base.rb

@@ -24,9 +24,21 @@ module Datadog
           # e.g. `settings :foo { option :bar }` --> `config.foo.bar`
           # @param [Symbol] name option name. Methods will be created based on this name.
           def settings(name, &block)
-            settings_class = new_settings_class(name, &block)
-
-            option(name) do |o|
+            nested_settings_path = settings_path ? "#{settings_path}.#{name}" : name.to_s
+            settings_class = new_settings_class(name, nested_settings_path, &block)
+            # Record the child settings class on the owning class so
+            # Options::ClassMethods#settings_path= can later propagate any path
+            # changes down to nested settings classes.
+            #
+            # Example:
+            #   settings :cache_key { option :enabled }
+            # creates a child class whose initial settings_path is "cache_key".
+            # If a contrib integration later assigns the parent path to
+            # "tracing.active_support", settings_children lets us update the
+            # nested class to "tracing.active_support.cache_key" as well.
+            settings_children[name] = settings_class
+
+            option(name, is_settings: true) do |o|
               o.default { settings_class.new }
 
               o.resetter do |value|
@@ -40,9 +52,9 @@ module Datadog
 
           private
 
-          def new_settings_class(name, &block)
+          def new_settings_class(name, settings_path, &block)
             Class.new { include Configuration::Base }.tap do |klass|
-              klass.instance_variable_set(:@settings_name, name)
+              klass.instance_variable_set(:@settings_path, settings_path)
               klass.instance_eval(&block) if block
             end
           end

data/lib/datadog/core/configuration/components.rb

@@ -12,6 +12,7 @@ require_relative '../telemetry/component'
 require_relative '../workers/runtime_metrics'
 require_relative '../remote/component'
 require_relative '../utils/at_fork_monkey_patch'
+require_relative '../utils/spawn_monkey_patch'
 require_relative '../utils/only_once'
 require_relative '../../tracing/component'
 require_relative '../../profiling/component'
@@ -22,6 +23,7 @@ require_relative '../../open_feature/component'
 require_relative '../../error_tracking/component'
 require_relative '../crashtracking/component'
 require_relative '../environment/agent_info'
+require_relative '../environment/identity'
 require_relative '../process_discovery'
 require_relative '../../data_streams/processor'
 
@@ -31,7 +33,7 @@ module Datadog
       # Global components for the trace library.
       class Components
         # Class-level constant to ensure fork patch is applied only once
-        AT_FORK_ONLY_ONCE = Utils::OnlyOnce.new
+        PATCH_ONLY_ONCE = Utils::OnlyOnce.new
 
         class << self
           def build_health_metrics(settings, logger, telemetry)
@@ -128,8 +130,11 @@ module Datadog
           Deprecations.log_deprecations_from_all_sources(@logger)
 
           # Register fork handling once globally
-          self.class::AT_FORK_ONLY_ONCE.run do
+          self.class::PATCH_ONLY_ONCE.run do
             Utils::AtForkMonkeyPatch.apply!
+            Utils::SpawnMonkeyPatch.apply!(
+              lineage_envs_provider: Core::Environment::Identity.method(:runtime_propagation_envs),
+            )
 
             # Register callback that calls Components.after_fork
             Utils::AtForkMonkeyPatch.at_fork(:child) do
@@ -172,6 +177,11 @@ module Datadog
 
           # Configure non-privileged components.
           Datadog::Tracing::Contrib::Component.configure(settings)
+
+          # Load the core Rails Railtie when Rails is present so all products benefit from Rails-specific setup.
+          if defined?(::Rails::Railtie)
+            require_relative '../contrib/rails/railtie'
+          end
         end
 
         # Called when a fork is detected
@@ -204,14 +214,13 @@ module Datadog
             end
           end
 
-          if settings.remote.enabled && old_state&.remote_started?
+          if remote && old_state&.remote_started?
             # The library was reconfigured and previously it already started
             # the remote component (i.e., it received at least one request
             # through the installed Rack middleware which started the remote).
             # If the new configuration also has remote enabled, start the
             # new remote right away.
-            # remote should always be not nil here but steep doesn't know this.
-            remote&.start
+            remote.start
           end
 
           # This should stay here, not in initialize. During reconfiguration, the order of the calls is:
@@ -279,11 +288,15 @@ module Datadog
           unused_statsd = (old_statsd - (old_statsd & new_statsd))
           unused_statsd.each(&:close)
 
-          # enqueue closing event before stopping telemetry so it will be sent out on shutdown
+          Core::ProcessDiscovery.shutdown!
+
+          # Shut down telemetry last so that all other components may
+          # report shutdown errors.
+          #
+          # Enqueue closing event before stopping telemetry so it will be
+          # sent out on shutdown.
           telemetry.emit_closing! unless replacement&.telemetry&.enabled
           telemetry.shutdown!
-
-          Core::ProcessDiscovery.shutdown!
         end
 
         # Returns the current state of various components.

data/lib/datadog/core/configuration/config_helper.rb

@@ -40,6 +40,12 @@ module Datadog
           !get_environment_variable(name).nil?
         end
 
+        # Returns the source environment as a Hash. Used when the full environment
+        # must be passed through (e.g. Process.spawn child env).
+        def to_h
+          @source_env.to_h
+        end
+
         alias_method :has_key?, :key?
         alias_method :include?, :key?
         alias_method :member?, :key?
@@ -97,4 +103,7 @@ module Datadog
       end
     end
   end
+  unless const_defined?(:DATADOG_ENV, false)
+    DATADOG_ENV = Core::Configuration::ConfigHelper.new
+  end
 end

data/lib/datadog/core/configuration/option.rb

@@ -77,6 +77,16 @@ module Datadog
           @precedence_set = Precedence::DEFAULT
         end
 
+        # Computes the name of the option with the settings path.
+        # E.g. "tracing.rails.middleware_names" for the "middleware_names" option in the "tracing.rails" settings.
+        # @return [String] the name of the option with the settings path
+        def name_with_settings_path
+          @name_with_settings_path ||= begin
+            settings_path = @context.class.settings_path
+            settings_path.nil? ? definition.name.to_s : "#{settings_path}.#{definition.name}"
+          end
+        end
+
         # Overrides the current value for this option if the `precedence` is equal or higher than
         # the previously set value.
         # The first call to `#set` will always store the value regardless of precedence.
@@ -89,7 +99,7 @@ module Datadog
             # This should be uncommon, as higher precedence values tend to
             # happen later in the application lifecycle.
             Datadog.logger.info do
-              "Option '#{definition.name}' not changed to '#{value}' (precedence: #{precedence.name}) because the higher " \
+              "Option '#{name_with_settings_path}' not changed to '#{value}' (precedence: #{precedence.name}) because the higher " \
                 "precedence value '#{@value}' (precedence: #{@precedence_set.name}) was already set."
             end
 
@@ -167,7 +177,8 @@ module Datadog
 
         def default_value
           if definition.default.instance_of?(Proc)
-            context_eval(&definition.default)
+            # Steep: https://github.com/soutaro/steep/issues/335
+            context_eval(&definition.default) # steep:ignore BlockTypeMismatch
           else
             definition.default_proc || Core::Utils::SafeDup.frozen_or_dup(definition.default)
           end
@@ -177,6 +188,21 @@ module Datadog
           precedence_set == Precedence::DEFAULT
         end
 
+        def settings?
+          @definition.is_settings
+        end
+
+        def values_per_precedence
+          # value_per_precedence is only filled after we call `get` once.
+          get unless @is_set
+
+          @value_per_precedence.each_with_object({}) do |(precedence, value), result|
+            next if value.equal?(UNSET)
+
+            result[precedence] = value
+          end
+        end
+
         private
 
         def coerce_env_variable(value)
@@ -215,7 +241,7 @@ module Datadog
             value
           else
             raise InvalidDefinitionError,
-              "The option #{@definition.name} is using an unsupported type option for env coercion `#{@definition.type}`"
+              "The option #{name_with_settings_path} is using an unsupported type option for env coercion `#{@definition.type}`"
           end
         end
 
@@ -238,10 +264,10 @@ module Datadog
 
           if raise_error
             error_msg = if @definition.type_options[:nilable]
-              "The setting `#{@definition.name}` inside your app's `Datadog.configure` block expects a " \
+              "The setting `#{name_with_settings_path}` inside your app's `Datadog.configure` block expects a " \
               "#{@definition.type} or `nil`, but a `#{value.class}` was provided (#{value.inspect})." \
             else
-              "The setting `#{@definition.name}` inside your app's `Datadog.configure` block expects a " \
+              "The setting `#{name_with_settings_path}` inside your app's `Datadog.configure` block expects a " \
               "#{@definition.type}, but a `#{value.class}` was provided (#{value.inspect})." \
             end
 
@@ -275,7 +301,7 @@ module Datadog
             true # No validation is performed when option is typeless
           else
             raise InvalidDefinitionError,
-              "The option #{@definition.name} is using an unsupported type option `#{@definition.type}`"
+              "The option #{name_with_settings_path} is using an unsupported type option `#{@definition.type}`"
           end
         end