datadog 2.39.0 → 2.47.0

data/lib/datadog/appsec/contrib/rails/gateway/watcher.rb

@@ -26,7 +26,7 @@ module Datadog
                   limit = Datadog.configuration.appsec.body_parsing_size_limit
 
                   persistent_data = {
-                    "server.request.path_params" => gateway_request.route_params
+                    "server.request.path_params" => gateway_request.route_params,
                   }
 
                   unless limit.zero?
@@ -66,10 +66,23 @@ module Datadog
                 gateway.watch("rails.response.body.json") do |stack, container|
                   context = container.context
 
+                  body_bytesize = container.data.bytesize
+                  limit = Datadog.configuration.appsec.body_parsing_size_limit
+
                   persistent_data = {
-                    "server.response.body" => container.data
+                    "server.response.body.byte_length" => body_bytesize,
                   }
-                  result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
+
+                  if limit > 0 && body_bytesize <= limit
+                    json = extract_json(container.data)
+                    persistent_data["server.response.body"] = WAF::JSONString.new(json) if json
+                  end
+
+                  result = begin
+                    context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
+                  rescue WAF::ConversionError
+                    next stack.call(container)
+                  end
 
                   if result.match?
                     context.events.push(
@@ -85,6 +98,20 @@ module Datadog
                   stack.call(container)
                 end
               end
+
+              private
+
+              def extract_json(body)
+                return body unless body.start_with?("/**/")
+                return unless body.end_with?(")")
+
+                # NOTE: Best-effort support for standard Rails JSONP callbacks.
+                #       Callback expressions with parentheses may prevent extraction
+                opening_parenthesis_index = body.b.index("(")
+                return unless opening_parenthesis_index
+
+                body.byteslice(opening_parenthesis_index + 1, body.bytesize - opening_parenthesis_index - 2) #: String
+              end
             end
           end
         end

data/lib/datadog/appsec/contrib/rails/patches/render_to_body_patch.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require_relative "../../../utils/hash_coercion"
 require_relative "../../../instrumentation/gateway/argument"
 
 module Datadog
@@ -8,8 +7,8 @@ module Datadog
     module Contrib
       module Rails
         module Patches
-          # A patch targeting `AbstractController::Rendering#render_to_body`
-          # method to capture JSON response body right before it is serialized.
+          # A patch for `AbstractController::Rendering#render_to_body` that captures
+          # the serialized JSON response body for AppSec inspection
           module RenderToBodyPatch
             def render_to_body(options = {})
               return super unless options.key?(:json)
@@ -17,13 +16,15 @@ module Datadog
               context = request.env[Datadog::AppSec::Ext::CONTEXT_KEY]
               return super unless context
 
-              data = Utils::HashCoercion.coerce(options[:json])
-              return super unless data
+              # NOTE: Custom renderers can return streaming bodies, which we
+              #       currently do not inspect
+              body = super
+              return body unless body.is_a?(String)
 
-              container = Instrumentation::Gateway::DataContainer.new(data, context: context)
+              container = Instrumentation::Gateway::DataContainer.new(body, context: context)
               Instrumentation.gateway.push("rails.response.body.json", container)
 
-              super
+              body
             end
           end
         end

data/lib/datadog/appsec/contrib/rails/request_middleware.rb

@@ -6,7 +6,8 @@ module Datadog
       module Rails
         # Rack middleware for AppSec on Rails
         class RequestMiddleware
-          def initialize(app, opt = {})
+          # TODO: opt is never used, it can probably be safely removed
+          def initialize(app, opt = {}) # steep:ignore DifferentMethodParameterKind
             @app = app
           end
 

data/lib/datadog/appsec/contrib/rest_client/request_ssrf_detection_patch.rb

@@ -24,7 +24,7 @@ module Datadog
             ephemeral_data = {
               "server.io.net.url" => url,
               "server.io.net.request.method" => method.to_s.upcase,
-              "server.io.net.request.headers" => headers
+              "server.io.net.request.headers" => headers,
             }
 
             is_redirect = context.state[:downstream_redirect_url] == url
@@ -68,7 +68,7 @@ module Datadog
             # @type var ephemeral_data: ::Datadog::AppSec::Context::input_data
             ephemeral_data = {
               "server.io.net.response.status" => response.code.to_s,
-              "server.io.net.response.headers" => headers
+              "server.io.net.response.headers" => headers,
             }
 
             is_redirect = REDIRECT_STATUS_CODES.cover?(response.code.to_i) && headers.key?("location")

data/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb

@@ -77,7 +77,7 @@ module Datadog
                   context = gateway_request.env[AppSec::Ext::CONTEXT_KEY] # : Context
 
                   persistent_data = {
-                    "server.request.path_params" => gateway_route_params.params
+                    "server.request.path_params" => gateway_route_params.params,
                   }
 
                   result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
@@ -101,10 +101,22 @@ module Datadog
                 gateway.watch("sinatra.response.body.json") do |stack, container|
                   context = container.context # : Context
 
+                  body_bytesize = container.data.bytesize
+                  limit = Datadog.configuration.appsec.body_parsing_size_limit
+
                   persistent_data = {
-                    "server.response.body" => container.data
+                    "server.response.body.byte_length" => body_bytesize,
                   }
-                  result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
+
+                  if limit > 0 && body_bytesize <= limit
+                    persistent_data["server.response.body"] = WAF::JSONString.new(container.data)
+                  end
+
+                  result = begin
+                    context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
+                  rescue WAF::ConversionError
+                    next stack.call(container)
+                  end
 
                   if result.match?
                     context.events.push(

data/lib/datadog/appsec/contrib/sinatra/patches/json_patch.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require_relative "../../../utils/hash_coercion"
 require_relative "../../../instrumentation/gateway/argument"
 
 module Datadog
@@ -8,20 +7,22 @@ module Datadog
     module Contrib
       module Sinatra
         module Patches
-          # A patch targeting `Sinatra::JSON#json` method to capture JSON response
-          # body right before it is serialized.
+          # A patch for `Sinatra::JSON#json` that captures the serialized
+          # response body for AppSec inspection
           module JsonPatch
             def json(object, options = {})
               context = @request.env[Datadog::AppSec::Ext::CONTEXT_KEY] # : Context?
               return super unless context
 
-              data = Utils::HashCoercion.coerce(object)
-              return super unless data
+              # NOTE: Custom renderers can return streaming bodies, which we
+              #       currently do not inspect
+              body = super
+              return body unless body.is_a?(String)
 
-              container = Instrumentation::Gateway::DataContainer.new(data, context: context)
+              container = Instrumentation::Gateway::DataContainer.new(body, context: context)
               Instrumentation.gateway.push("sinatra.response.body.json", container)
 
-              super
+              body
             end
           end
         end

data/lib/datadog/appsec/contrib/sinatra/request_middleware.rb

@@ -6,7 +6,8 @@ module Datadog
       module Sinatra
         # Rack middleware for AppSec on Sinatra
         class RequestMiddleware
-          def initialize(app, opt = {})
+          # TODO: opt is never used, it can probably be safely removed
+          def initialize(app, opt = {}) # steep:ignore DifferentMethodParameterKind
             @app = app
           end
 

data/lib/datadog/appsec/processor/rule_loader.rb

@@ -59,7 +59,7 @@ module Datadog
             {
               "id" => id,
               "type" => "data_with_expiration",
-              "data" => denylist.map { |v| {"value" => v.to_s, "expiration" => 2**63} }
+              "data" => denylist.map { |v| {"value" => v.to_s, "expiration" => 2**63} },
             }
           end
 
@@ -82,12 +82,12 @@ module Datadog
                   "parameters" => {
                     "inputs" => [
                       {
-                        "address" => "http.client_ip"
-                      }
+                        "address" => "http.client_ip",
+                      },
                     ],
-                    "list" => pass
-                  }
-                }
+                    "list" => pass,
+                  },
+                },
               ],
               "id" => SecureRandom.uuid,
             }
@@ -99,15 +99,15 @@ module Datadog
                   "parameters" => {
                     "inputs" => [
                       {
-                        "address" => "http.client_ip"
-                      }
+                        "address" => "http.client_ip",
+                      },
                     ],
-                    "list" => monitor
-                  }
-                }
+                    "list" => monitor,
+                  },
+                },
               ],
               "id" => SecureRandom.uuid,
-              "on_match" => "monitor"
+              "on_match" => "monitor",
             }
 
             exclusions

data/lib/datadog/appsec/response.rb

@@ -43,7 +43,7 @@ module Datadog
               content(
                 security_response_id: interrupt_params["security_response_id"],
                 content_type: content_type
-              )
+              ),
             ],
           )
         end

data/lib/datadog/appsec/route_normalizer/route_pattern.rb

@@ -31,7 +31,7 @@ module Datadog
         OPTIONAL_GROUP_SIGILS = [
           GROUP_OPEN_CHAR,
           GROUP_CLOSE_CHAR,
-          OPTIONAL_GROUP_SUFFIX_CHAR
+          OPTIONAL_GROUP_SUFFIX_CHAR,
         ].join
 
         NAMED_PARAM_PREFIX_CHAR = ":"
@@ -41,14 +41,14 @@ module Datadog
           NAMED_PARAM_PREFIX_CHAR,
           GLOB_PARAM_PREFIX_CHAR,
           GROUP_OPEN_CHAR,
-          GROUP_CLOSE_CHAR
+          GROUP_CLOSE_CHAR,
         ].join
 
         PARAM_NAME_CHARS = [*"a".."z", *"A".."Z", *"0".."9", "_"].join
 
         EXCLUDED_PARAM_NAME_TERMINATOR_CHARS = [
           PARAM_NAME_CHARS,
-          PATTERN_STRUCTURE_CHARS
+          PATTERN_STRUCTURE_CHARS,
         ].join
 
         MAX_RESOLVE_LENGTH = 8192

data/lib/datadog/appsec/security_engine/engine.rb

@@ -31,7 +31,7 @@ module Datadog
           @waf_builder = WAF::HandleBuilder.new(
             obfuscator: {
               key_regex: appsec_settings.obfuscator_key_regex,
-              value_regex: appsec_settings.obfuscator_value_regex
+              value_regex: appsec_settings.obfuscator_value_regex,
             }
           )
 
@@ -71,7 +71,8 @@ module Datadog
           if is_ruleset_update &&
               (diagnostics.key?("error") ||
               diagnostics.dig("rules", "error") ||
-              diagnostics.dig("processors", "errors"))
+              diagnostics.dig("processors", "error"))
+            @waf_builder.remove_config_at_path(path)
             diagnostics = load_default_config(telemetry: AppSec.telemetry)
             @reconfigured_ruleset_version = diagnostics["ruleset_version"]
             report_configuration_diagnostics(diagnostics, action: "update", telemetry: AppSec.telemetry)
@@ -130,7 +131,7 @@ module Datadog
             tags: {
               waf_version: Datadog::AppSec::WAF::VERSION::BASE_STRING,
               event_rules_version: ruleset_version.to_s,
-              success: success.to_s
+              success: success.to_s,
             }
           )
         end
@@ -156,7 +157,7 @@ module Datadog
           common_tags = {
             waf_version: Datadog::AppSec::WAF::VERSION::BASE_STRING,
             event_rules_version: diagnostics["ruleset_version"].to_s,
-            action: action
+            action: action,
           }
 
           if diagnostics["error"]

data/lib/datadog/core/configuration/components.rb

@@ -21,7 +21,7 @@ require_relative "../../ai_guard/component"
 require_relative "../../di/component"
 require_relative "../../symbol_database"
 require_relative "../../symbol_database/component"
-require_relative "../../open_feature/component"
+require_relative "../../open_feature/activation"
 require_relative "../../error_tracking/component"
 require_relative "../crashtracking/component"
 require_relative "../environment/agent_info"
@@ -55,7 +55,7 @@ module Datadog
           # extension, remote config off, non-MRI). Under the always-build model
           # it is non-nil even when the DI setting defaults to off, so a built
           # SymbolDatabase component is inert until an upload is actually
-          # permitted: the component's own gate (see Component#upload_allowed?)
+          # permitted: the component's own gate, SymbolDatabase::Component#upload_allowed?,
           # only extracts and uploads when DI is truly active or the customer
           # opted in explicitly. This mirrors DI advertising/building a component
           # by default while installing no probes until it is enabled.
@@ -154,8 +154,7 @@ module Datadog
           :ai_guard,
           :agent_info,
           :data_streams,
-          :symbol_database,
-          :open_feature
+          :symbol_database
 
         def initialize(settings)
           @settings = settings
@@ -188,8 +187,14 @@ module Datadog
 
           @telemetry = self.class.build_telemetry(settings, agent_settings, @logger)
 
-          @remote = Remote::Component.build(settings, agent_settings, logger: @logger, telemetry: telemetry)
+          @remote = Remote::Component.build(
+            settings,
+            agent_settings,
+            logger: @logger,
+            telemetry: telemetry,
+          )
           @tracer = Datadog::Tracing::Component.build_tracer(settings, agent_settings, logger: @logger)
+          Datadog::Tracing::Component.bind_transport_telemetry(@tracer, telemetry)
           @crashtracker = self.class.build_crashtracker(settings, agent_settings, logger: @logger)
 
           @profiler, profiler_logger_extra = Datadog::Profiling::Component.build_profiler_component(
@@ -204,7 +209,13 @@ module Datadog
           @health_metrics = self.class.build_health_metrics(settings, @logger, telemetry)
           @appsec = Datadog::AppSec::Component.build_appsec_component(settings, telemetry: telemetry)
           @ai_guard = Datadog::AIGuard::Component.build(settings, logger: @logger, telemetry: telemetry)
-          @open_feature = OpenFeature::Component.build(settings, agent_settings, logger: @logger, telemetry: telemetry)
+          @open_feature_activation = OpenFeature::Activation.new(
+            settings,
+            agent_settings,
+            remote,
+            logger: @logger,
+            telemetry: telemetry,
+          )
           @dynamic_instrumentation = Datadog::DI::Component.build(settings, agent_settings, @logger, telemetry: telemetry)
           # Only build symbol database when enabled, so a disabled component is
           # never constructed.
@@ -243,11 +254,14 @@ module Datadog
 
         # Called when a fork is detected
         def after_fork
+          tracer.after_fork
           telemetry.after_fork
           remote&.after_fork
           crashtracker&.update_on_fork
           ProcessDiscovery.after_fork
           symbol_database&.after_fork!
+          data_streams&.restart_flush_thread
+          @open_feature_activation.after_fork
         end
 
         # Hot-swaps with a new sampler.
@@ -262,6 +276,8 @@ module Datadog
         def startup!(settings, old_state: nil)
           telemetry.start(old_state&.telemetry_enabled?, components: self)
 
+          @open_feature_activation.startup!
+
           if settings.profiling.enabled
             if profiler
               profiler.start
@@ -272,12 +288,20 @@ module Datadog
             end
           end
 
+          if dynamic_instrumentation && old_state
+            dynamic_instrumentation.record_rc_enablement_signal_paths(old_state.rc_enablement_signal_paths)
+          end
+
           if remote && old_state&.remote_started?
             # The library was reconfigured and previously it already started
             # the remote component (i.e., it received at least one request
             # through the installed Rack middleware which started the remote).
             # If the new configuration also has remote enabled, start the
             # new remote right away.
+            #
+            # The DI signal-path restore above must precede this: the
+            # replacement client's first sync can dispatch the APM_TRACING
+            # merge, whose withdrawal branch reads the recorded paths.
             remote.start
           end
 
@@ -320,8 +344,14 @@ module Datadog
           # Shutdown Symbol Database
           symbol_database&.shutdown!
 
-          # Shutdown OpenFeature component
-          open_feature&.shutdown!
+          begin
+            @open_feature_activation.shutdown!
+          rescue => e
+            # Feature Flags is optional and must never interrupt library shutdown.
+            description = "Feature Flags delivery failed to shut down"
+            logger.error("#{description}: #{e.class}: #{e.message}")
+            telemetry.report(e, description: description)
+          end
 
           # Decommission AppSec
           appsec&.shutdown!
@@ -352,13 +382,13 @@ module Datadog
           # have direct ownership of it.
           old_statsd = [
             runtime_metrics.metrics.statsd,
-            health_metrics.statsd
+            health_metrics.statsd,
           ].compact.uniq
 
           new_statsd = if replacement
             [
               replacement.runtime_metrics.metrics.statsd,
-              replacement.health_metrics.statsd
+              replacement.health_metrics.statsd,
             ].compact.uniq
           else
             []
@@ -383,21 +413,22 @@ module Datadog
           # di_implicitly_enabled distinguishes RC-driven start from explicit
           # start (env var or programmatic) so that an explicit
           # `enabled = false` on reconfiguration can take effect.
-          #
-          # using_default? — not `!enabled` — because Datadog.configure has
-          # already mutated the singleton settings by the time #state runs,
-          # so `!enabled` would treat a customer's explicit
-          # `enabled = false` as "implicitly enabled" and restart DI.
-          # using_default? asks "did the customer ever touch this setting",
-          # which is the right question for RC carry-over.
-          di_implicit = dynamic_instrumentation&.started? &&
-            @settings.dynamic_instrumentation.using_default?(:enabled)
+          di_implicit = dynamic_instrumentation&.implicitly_enabled?
           ComponentsState.new(
             telemetry_enabled: telemetry.enabled,
             remote_started: remote&.started?,
             di_implicitly_enabled: di_implicit || false,
+            rc_enablement_signal_paths: dynamic_instrumentation&.rc_enablement_signal_paths || [],
           )
         end
+
+        def open_feature
+          @open_feature_activation.component
+        end
+
+        private
+
+        attr_reader :open_feature_activation
       end
     end
   end

data/lib/datadog/core/configuration/components_state.rb

@@ -5,10 +5,12 @@ module Datadog
     module Configuration
       # Stores the state of component tree when replacing the tree.
       class ComponentsState
-        def initialize(telemetry_enabled:, remote_started:, di_implicitly_enabled: false)
+        def initialize(telemetry_enabled:, remote_started:, di_implicitly_enabled: false,
+          rc_enablement_signal_paths: [])
           @telemetry_enabled = !!telemetry_enabled
           @remote_started = !!remote_started
           @di_implicitly_enabled = !!di_implicitly_enabled
+          @rc_enablement_signal_paths = rc_enablement_signal_paths || []
         end
 
         def telemetry_enabled?
@@ -22,6 +24,11 @@ module Datadog
         def di_implicitly_enabled?
           @di_implicitly_enabled
         end
+
+        # The RC enable-signal carrier paths recorded by the replaced tree's DI component.
+        #
+        # @return [Array<Core::Remote::Configuration::Path>]
+        attr_reader :rc_enablement_signal_paths
       end
     end
   end

data/lib/datadog/core/configuration/option_definition.rb

@@ -165,7 +165,7 @@ module Datadog
               resetter: @resetter,
               setter: @setter,
               type: @type,
-              type_options: @type_options
+              type_options: @type_options,
             }
           end
 

data/lib/datadog/core/configuration/options.rb

@@ -84,7 +84,7 @@ module Datadog
             end
             {
               option_name.to_sym => opt_getter,
-              :"#{option_name}=" => opt_setter
+              :"#{option_name}=" => opt_setter,
             }
           end
 

data/lib/datadog/core/configuration/settings.rb

@@ -644,6 +644,13 @@ module Datadog
               o.env "DD_PROFILING_EXPERIMENTAL_USE_SYSTEM_DNS"
               o.default true
             end
+
+            # Whether the profiler samples include the class/module name in stack frames, like `Foo::Bar#baz`, or not (`baz`)
+            option :experimental_show_classes_enabled do |o|
+              o.type :bool
+              o.env "DD_PROFILING_EXPERIMENTAL_SHOW_CLASSES_ENABLED"
+              o.default false
+            end
           end
 
           # @public_api
@@ -750,12 +757,15 @@ module Datadog
                 val ||= ""
                 # maps OpenTelemetry semantic attributes to Datadog tags
                 key = case key.downcase
-                when "deployment.environment" then "env"
+                when "deployment.environment.name"
+                  result["env"] = val
+                  nil
+                when "deployment.environment" then "env" unless result.key?("env")
                 when "service.version" then "version"
                 when "service.name" then "service"
                 else key
                 end
-                result[key] = val unless key.empty?
+                result[key] = val unless key.nil? || key.empty?
               end
             end
             result
@@ -792,55 +802,40 @@ module Datadog
           end
         end
 
-        # The time provider used by Datadog. It must respect the interface of [Time](https://ruby-doc.org/core-3.0.1/Time.html).
-        #
-        # When testing, it can be helpful to use a different time provider.
-        #
-        # For [Timecop](https://rubygems.org/gems/timecop), for example, `->{ Time.now_without_mock_time }`
-        # allows Datadog features to use the real wall time when time is frozen.
-        #
-        # @default `->{ Time.now }`
-        # @return [Proc<Time>]
+        # Deprecated and no longer does anything.
+        # This gem already takes care to not be affected by timecop.
+        # DEV-3.0: Remove `time_now_provider` option
         option :time_now_provider do |o|
-          o.default_proc { ::Time.now }
+          o.default_proc { Datadog::Core::Utils::Time.now }
           o.type :proc
-
-          o.after_set do |time_provider|
-            Core::Utils::Time.now_provider = time_provider
-          end
-
-          o.resetter do |_value|
-            # TODO: Resetter needs access to the default value
-            # TODO: to help reduce duplication.
-            -> { ::Time.now }.tap do |default|
-              Core::Utils::Time.now_provider = default
+          o.after_set do |_, _, precedence|
+            unless precedence == Datadog::Core::Configuration::Option::Precedence::DEFAULT
+              unless defined?(Datadog::CI) # the datadog-ci gem needs to work with any datadog gem version and not warn
+                Core.log_deprecation(key: :time_now_provider) do
+                  "The time_now_provider setting has been deprecated for removal " \
+                    "and no longer does anything. Please remove it from your Datadog.configure block. " \
+                    "The datadog gem always uses the real non-mocked time, even when the timecop gem monkey-patches Time."
+                end
+              end
             end
           end
         end
 
-        # The monotonic clock time provider used by Datadog. This option is internal and is used by `datadog-ci`
-        # gem to avoid traces' durations being skewed by timecop.
-        #
-        # It must respect the interface of [Datadog::Core::Utils::Time#get_time] method.
-        #
-        # For [Timecop](https://rubygems.org/gems/timecop), for example,
-        # `->(unit = :float_second) { ::Process.clock_gettime_without_mock(Datadog::Core::Utils::Time::MONOTONIC_CLOCK_ID, unit) }`
-        # allows Datadog features to use the real monotonic time when time is frozen with
-        # `Timecop.mock_process_clock = true`.
-        #
-        # @default `->(unit = :float_second) { ::Process.clock_gettime(Core::Utils::Time::MONOTONIC_CLOCK_ID, unit) }`
-        # @return [Proc<Numeric>]
+        # Deprecated and no longer does anything.
+        # This gem already takes care to not be affected by timecop.
+        # DEV-3.0: Remove `get_time_provider` option
         option :get_time_provider do |o|
-          o.default_proc { |unit = :float_second| ::Process.clock_gettime(Core::Utils::Time::MONOTONIC_CLOCK_ID, unit) }
+          o.default_proc { |unit = :float_second| Datadog::Core::Utils::Time.get_time(unit) }
           o.type :proc
-
-          o.after_set do |get_time_provider|
-            Core::Utils::Time.get_time_provider = get_time_provider
-          end
-
-          o.resetter do |_value|
-            ->(unit = :float_second) { ::Process.clock_gettime(Core::Utils::Time::MONOTONIC_CLOCK_ID, unit) }.tap do |default|
-              Core::Utils::Time.get_time_provider = default
+          o.after_set do |_, _, precedence|
+            unless precedence == Datadog::Core::Configuration::Option::Precedence::DEFAULT
+              unless defined?(Datadog::CI) # the datadog-ci gem needs to work with any datadog gem version and not warn
+                Core.log_deprecation(key: :get_time_provider) do
+                  "The get_time_provider setting has been deprecated for removal " \
+                    "and no longer does anything. Please remove it from your Datadog.configure block. " \
+                    "The datadog gem always uses the real non-mocked time, even when the timecop gem monkey-patches Process.clock_gettime."
+                end
+              end
             end
           end
         end