datadog 2.45.0 → 2.47.0

data/lib/datadog/appsec/contrib/rails/gateway/watcher.rb

@@ -66,10 +66,23 @@ module Datadog
                 gateway.watch("rails.response.body.json") do |stack, container|
                   context = container.context
 
+                  body_bytesize = container.data.bytesize
+                  limit = Datadog.configuration.appsec.body_parsing_size_limit
+
                   persistent_data = {
-                    "server.response.body" => container.data,
+                    "server.response.body.byte_length" => body_bytesize,
                   }
-                  result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
+
+                  if limit > 0 && body_bytesize <= limit
+                    json = extract_json(container.data)
+                    persistent_data["server.response.body"] = WAF::JSONString.new(json) if json
+                  end
+
+                  result = begin
+                    context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
+                  rescue WAF::ConversionError
+                    next stack.call(container)
+                  end
 
                   if result.match?
                     context.events.push(
@@ -85,6 +98,20 @@ module Datadog
                   stack.call(container)
                 end
               end
+
+              private
+
+              def extract_json(body)
+                return body unless body.start_with?("/**/")
+                return unless body.end_with?(")")
+
+                # NOTE: Best-effort support for standard Rails JSONP callbacks.
+                #       Callback expressions with parentheses may prevent extraction
+                opening_parenthesis_index = body.b.index("(")
+                return unless opening_parenthesis_index
+
+                body.byteslice(opening_parenthesis_index + 1, body.bytesize - opening_parenthesis_index - 2) #: String
+              end
             end
           end
         end

data/lib/datadog/appsec/contrib/rails/patches/render_to_body_patch.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require_relative "../../../utils/hash_coercion"
 require_relative "../../../instrumentation/gateway/argument"
 
 module Datadog
@@ -8,8 +7,8 @@ module Datadog
     module Contrib
       module Rails
         module Patches
-          # A patch targeting `AbstractController::Rendering#render_to_body`
-          # method to capture JSON response body right before it is serialized.
+          # A patch for `AbstractController::Rendering#render_to_body` that captures
+          # the serialized JSON response body for AppSec inspection
           module RenderToBodyPatch
             def render_to_body(options = {})
               return super unless options.key?(:json)
@@ -17,13 +16,15 @@ module Datadog
               context = request.env[Datadog::AppSec::Ext::CONTEXT_KEY]
               return super unless context
 
-              data = Utils::HashCoercion.coerce(options[:json])
-              return super unless data
+              # NOTE: Custom renderers can return streaming bodies, which we
+              #       currently do not inspect
+              body = super
+              return body unless body.is_a?(String)
 
-              container = Instrumentation::Gateway::DataContainer.new(data, context: context)
+              container = Instrumentation::Gateway::DataContainer.new(body, context: context)
               Instrumentation.gateway.push("rails.response.body.json", container)
 
-              super
+              body
             end
           end
         end

data/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb

@@ -101,10 +101,22 @@ module Datadog
                 gateway.watch("sinatra.response.body.json") do |stack, container|
                   context = container.context # : Context
 
+                  body_bytesize = container.data.bytesize
+                  limit = Datadog.configuration.appsec.body_parsing_size_limit
+
                   persistent_data = {
-                    "server.response.body" => container.data,
+                    "server.response.body.byte_length" => body_bytesize,
                   }
-                  result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
+
+                  if limit > 0 && body_bytesize <= limit
+                    persistent_data["server.response.body"] = WAF::JSONString.new(container.data)
+                  end
+
+                  result = begin
+                    context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
+                  rescue WAF::ConversionError
+                    next stack.call(container)
+                  end
 
                   if result.match?
                     context.events.push(

data/lib/datadog/appsec/contrib/sinatra/patches/json_patch.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require_relative "../../../utils/hash_coercion"
 require_relative "../../../instrumentation/gateway/argument"
 
 module Datadog
@@ -8,20 +7,22 @@ module Datadog
     module Contrib
       module Sinatra
         module Patches
-          # A patch targeting `Sinatra::JSON#json` method to capture JSON response
-          # body right before it is serialized.
+          # A patch for `Sinatra::JSON#json` that captures the serialized
+          # response body for AppSec inspection
           module JsonPatch
             def json(object, options = {})
               context = @request.env[Datadog::AppSec::Ext::CONTEXT_KEY] # : Context?
               return super unless context
 
-              data = Utils::HashCoercion.coerce(object)
-              return super unless data
+              # NOTE: Custom renderers can return streaming bodies, which we
+              #       currently do not inspect
+              body = super
+              return body unless body.is_a?(String)
 
-              container = Instrumentation::Gateway::DataContainer.new(data, context: context)
+              container = Instrumentation::Gateway::DataContainer.new(body, context: context)
               Instrumentation.gateway.push("sinatra.response.body.json", container)
 
-              super
+              body
             end
           end
         end

data/lib/datadog/appsec/security_engine/engine.rb

@@ -71,7 +71,8 @@ module Datadog
           if is_ruleset_update &&
               (diagnostics.key?("error") ||
               diagnostics.dig("rules", "error") ||
-              diagnostics.dig("processors", "errors"))
+              diagnostics.dig("processors", "error"))
+            @waf_builder.remove_config_at_path(path)
             diagnostics = load_default_config(telemetry: AppSec.telemetry)
             @reconfigured_ruleset_version = diagnostics["ruleset_version"]
             report_configuration_diagnostics(diagnostics, action: "update", telemetry: AppSec.telemetry)

data/lib/datadog/core/configuration/components.rb

@@ -21,7 +21,7 @@ require_relative "../../ai_guard/component"
 require_relative "../../di/component"
 require_relative "../../symbol_database"
 require_relative "../../symbol_database/component"
-require_relative "../../open_feature/component"
+require_relative "../../open_feature/activation"
 require_relative "../../error_tracking/component"
 require_relative "../crashtracking/component"
 require_relative "../environment/agent_info"
@@ -55,7 +55,7 @@ module Datadog
           # extension, remote config off, non-MRI). Under the always-build model
           # it is non-nil even when the DI setting defaults to off, so a built
           # SymbolDatabase component is inert until an upload is actually
-          # permitted: the component's own gate (see Component#upload_allowed?)
+          # permitted: the component's own gate, SymbolDatabase::Component#upload_allowed?,
           # only extracts and uploads when DI is truly active or the customer
           # opted in explicitly. This mirrors DI advertising/building a component
           # by default while installing no probes until it is enabled.
@@ -154,8 +154,7 @@ module Datadog
           :ai_guard,
           :agent_info,
           :data_streams,
-          :symbol_database,
-          :open_feature
+          :symbol_database
 
         def initialize(settings)
           @settings = settings
@@ -188,15 +187,14 @@ module Datadog
 
           @telemetry = self.class.build_telemetry(settings, agent_settings, @logger)
 
-          # Bind Remote Configuration dispatch to this tree, which starts before it becomes the global Components instance.
           @remote = Remote::Component.build(
             settings,
             agent_settings,
             logger: @logger,
             telemetry: telemetry,
-            open_feature_component_provider: -> { @open_feature },
           )
           @tracer = Datadog::Tracing::Component.build_tracer(settings, agent_settings, logger: @logger)
+          Datadog::Tracing::Component.bind_transport_telemetry(@tracer, telemetry)
           @crashtracker = self.class.build_crashtracker(settings, agent_settings, logger: @logger)
 
           @profiler, profiler_logger_extra = Datadog::Profiling::Component.build_profiler_component(
@@ -211,7 +209,13 @@ module Datadog
           @health_metrics = self.class.build_health_metrics(settings, @logger, telemetry)
           @appsec = Datadog::AppSec::Component.build_appsec_component(settings, telemetry: telemetry)
           @ai_guard = Datadog::AIGuard::Component.build(settings, logger: @logger, telemetry: telemetry)
-          @open_feature = OpenFeature::Component.build(settings, agent_settings, logger: @logger, telemetry: telemetry)
+          @open_feature_activation = OpenFeature::Activation.new(
+            settings,
+            agent_settings,
+            remote,
+            logger: @logger,
+            telemetry: telemetry,
+          )
           @dynamic_instrumentation = Datadog::DI::Component.build(settings, agent_settings, @logger, telemetry: telemetry)
           # Only build symbol database when enabled, so a disabled component is
           # never constructed.
@@ -257,6 +261,7 @@ module Datadog
           ProcessDiscovery.after_fork
           symbol_database&.after_fork!
           data_streams&.restart_flush_thread
+          @open_feature_activation.after_fork
         end
 
         # Hot-swaps with a new sampler.
@@ -271,6 +276,8 @@ module Datadog
         def startup!(settings, old_state: nil)
           telemetry.start(old_state&.telemetry_enabled?, components: self)
 
+          @open_feature_activation.startup!
+
           if settings.profiling.enabled
             if profiler
               profiler.start
@@ -281,12 +288,20 @@ module Datadog
             end
           end
 
+          if dynamic_instrumentation && old_state
+            dynamic_instrumentation.record_rc_enablement_signal_paths(old_state.rc_enablement_signal_paths)
+          end
+
           if remote && old_state&.remote_started?
             # The library was reconfigured and previously it already started
             # the remote component (i.e., it received at least one request
             # through the installed Rack middleware which started the remote).
             # If the new configuration also has remote enabled, start the
             # new remote right away.
+            #
+            # The DI signal-path restore above must precede this: the
+            # replacement client's first sync can dispatch the APM_TRACING
+            # merge, whose withdrawal branch reads the recorded paths.
             remote.start
           end
 
@@ -329,8 +344,14 @@ module Datadog
           # Shutdown Symbol Database
           symbol_database&.shutdown!
 
-          # Shutdown OpenFeature component
-          open_feature&.shutdown!
+          begin
+            @open_feature_activation.shutdown!
+          rescue => e
+            # Feature Flags is optional and must never interrupt library shutdown.
+            description = "Feature Flags delivery failed to shut down"
+            logger.error("#{description}: #{e.class}: #{e.message}")
+            telemetry.report(e, description: description)
+          end
 
           # Decommission AppSec
           appsec&.shutdown!
@@ -392,21 +413,22 @@ module Datadog
           # di_implicitly_enabled distinguishes RC-driven start from explicit
           # start (env var or programmatic) so that an explicit
           # `enabled = false` on reconfiguration can take effect.
-          #
-          # using_default? — not `!enabled` — because Datadog.configure has
-          # already mutated the singleton settings by the time #state runs,
-          # so `!enabled` would treat a customer's explicit
-          # `enabled = false` as "implicitly enabled" and restart DI.
-          # using_default? asks "did the customer ever touch this setting",
-          # which is the right question for RC carry-over.
-          di_implicit = dynamic_instrumentation&.started? &&
-            @settings.dynamic_instrumentation.using_default?(:enabled)
+          di_implicit = dynamic_instrumentation&.implicitly_enabled?
           ComponentsState.new(
             telemetry_enabled: telemetry.enabled,
             remote_started: remote&.started?,
             di_implicitly_enabled: di_implicit || false,
+            rc_enablement_signal_paths: dynamic_instrumentation&.rc_enablement_signal_paths || [],
           )
         end
+
+        def open_feature
+          @open_feature_activation.component
+        end
+
+        private
+
+        attr_reader :open_feature_activation
       end
     end
   end

data/lib/datadog/core/configuration/components_state.rb

@@ -5,10 +5,12 @@ module Datadog
     module Configuration
       # Stores the state of component tree when replacing the tree.
       class ComponentsState
-        def initialize(telemetry_enabled:, remote_started:, di_implicitly_enabled: false)
+        def initialize(telemetry_enabled:, remote_started:, di_implicitly_enabled: false,
+          rc_enablement_signal_paths: [])
           @telemetry_enabled = !!telemetry_enabled
           @remote_started = !!remote_started
           @di_implicitly_enabled = !!di_implicitly_enabled
+          @rc_enablement_signal_paths = rc_enablement_signal_paths || []
         end
 
         def telemetry_enabled?
@@ -22,6 +24,11 @@ module Datadog
         def di_implicitly_enabled?
           @di_implicitly_enabled
         end
+
+        # The RC enable-signal carrier paths recorded by the replaced tree's DI component.
+        #
+        # @return [Array<Core::Remote::Configuration::Path>]
+        attr_reader :rc_enablement_signal_paths
       end
     end
   end

data/lib/datadog/core/configuration/supported_configurations.rb

@@ -51,6 +51,7 @@ module Datadog
           "DD_DBM_INJECT_SQL_BASEHASH",
           "DD_DBM_PROPAGATION_MODE",
           "DD_DISABLE_DATADOG_RAILS",
+          "DD_DYNAMIC_INSTRUMENTATION_CAPTURE_TIMEOUT_MS",
           "DD_DYNAMIC_INSTRUMENTATION_ENABLED",
           "DD_DYNAMIC_INSTRUMENTATION_MAX_TIME_TO_SERIALIZE",
           "DD_DYNAMIC_INSTRUMENTATION_PROBE_FILE",
@@ -71,6 +72,7 @@ module Datadog
           "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS",
           "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS",
           "DD_FEATURE_FLAGS_ENABLED",
+          "DD_FEATURE_FLAGS_EVALUATION_TELEMETRY_PER_SUBJECT_OPT_OUT_ENABLED",
           "DD_FLAGGING_EVALUATION_COUNTS_ENABLED",
           "DD_GIT_COMMIT_SHA",
           "DD_GIT_REPOSITORY_URL",
@@ -241,6 +243,7 @@ module Datadog
           "DD_TRACE_HTTP_ANALYTICS_ENABLED",
           "DD_TRACE_HTTP_ANALYTICS_SAMPLE_RATE",
           "DD_TRACE_HTTP_CLIENT_ERROR_STATUSES",
+          "DD_TRACE_HTTP_CLIENT_RESOURCE_NAME_QUANTIZE",
           "DD_TRACE_HTTP_DISTRIBUTED_TRACING",
           "DD_TRACE_HTTP_ENABLED",
           "DD_TRACE_HTTP_ERROR_STATUS_CODES",
@@ -399,6 +402,7 @@ module Datadog
 
       ALIASES =
         {"DD_DISABLE_DATADOG_RAILS" => ["DISABLE_DATADOG_RAILS"],
+         "DD_DYNAMIC_INSTRUMENTATION_CAPTURE_TIMEOUT_MS" => ["DD_DYNAMIC_INSTRUMENTATION_CAPTURE_TIMEOUT"],
          "DD_PROFILING_GVL_ENABLED" => ["DD_PROFILING_PREVIEW_GVL_ENABLED"],
          "DD_RUNTIME_METRICS_RUNTIME_ID_ENABLED" => ["DD_TRACE_EXPERIMENTAL_RUNTIME_ID_ENABLED"],
          "DD_SERVICE" => ["OTEL_SERVICE_NAME"],
@@ -409,10 +413,11 @@ module Datadog
          "DD_TRACE_SAMPLE_RATE" => ["OTEL_TRACES_SAMPLER"]}.freeze
 
       DEPRECATIONS =
-        Set["DD_PROFILING_PREVIEW_GVL_ENABLED", "DISABLE_DATADOG_RAILS"]
+        Set["DD_DYNAMIC_INSTRUMENTATION_CAPTURE_TIMEOUT", "DD_PROFILING_PREVIEW_GVL_ENABLED", "DISABLE_DATADOG_RAILS"]
 
       ALIAS_TO_CANONICAL =
-        {"DD_PROFILING_PREVIEW_GVL_ENABLED" => "DD_PROFILING_GVL_ENABLED",
+        {"DD_DYNAMIC_INSTRUMENTATION_CAPTURE_TIMEOUT" => "DD_DYNAMIC_INSTRUMENTATION_CAPTURE_TIMEOUT_MS",
+         "DD_PROFILING_PREVIEW_GVL_ENABLED" => "DD_PROFILING_GVL_ENABLED",
          "DD_TRACE_EXPERIMENTAL_RUNTIME_ID_ENABLED" => "DD_RUNTIME_METRICS_RUNTIME_ID_ENABLED",
          "DISABLE_DATADOG_RAILS" => "DD_DISABLE_DATADOG_RAILS",
          "OTEL_LOG_LEVEL" => "DD_TRACE_DEBUG",

data/lib/datadog/core/process_discovery.rb

@@ -23,7 +23,11 @@ module Datadog
           metadata = get_metadata(settings)
 
           shutdown!
-          @file_descriptor = _native_store_tracer_metadata(Datadog.logger, **metadata)
+          @file_descriptor = _native_store_tracer_metadata(
+            Datadog.logger,
+            settings.tracing.otel_thread_context_enabled,
+            **metadata
+          )
         end
 
         def shutdown!

data/lib/datadog/core/remote/client/capabilities.rb

@@ -6,7 +6,6 @@ require_relative "../../../tracing/remote"
 require_relative "../../../di/remote"
 require_relative "../../../symbol_database"
 require_relative "../../../symbol_database/remote"
-require_relative "../../../open_feature/remote"
 
 module Datadog
   module Core
@@ -14,40 +13,57 @@ module Datadog
       class Client
         # Capabilities
         class Capabilities
-          attr_reader :capabilities, :receivers, :base64_capabilities
-
-          def initialize(settings, telemetry, open_feature_component_provider: nil)
-            open_feature_component_provider ||= -> {}
+          def initialize(settings, telemetry:)
             @capabilities = []
             @products = []
-            @products_mutex = Mutex.new
             @receivers = []
             @telemetry = telemetry
+            @mutex = Mutex.new
+
+            register(settings)
 
-            register(settings, open_feature_component_provider)
+            @base64_capabilities = capabilities_to_base64(@capabilities)
+          end
 
-            @base64_capabilities = capabilities_to_base64
+          def capabilities
+            @mutex.synchronize { @capabilities.dup }
           end
 
           def products
-            @products_mutex.synchronize { @products.dup }
+            @mutex.synchronize { @products.dup }
           end
 
-          def add_products(*products)
-            @products_mutex.synchronize do
-              products.each { |product| @products << product unless @products.include?(product) }
+          def receivers
+            @mutex.synchronize { @receivers.dup }
+          end
+
+          def base64_capabilities
+            @mutex.synchronize { @base64_capabilities }
+          end
+
+          def register_runtime(capabilities:, products:, receivers:)
+            @mutex.synchronize do
+              register_capabilities(capabilities)
+              register_products(products)
+              register_receivers(receivers)
+              @base64_capabilities = capabilities_to_base64(@capabilities)
             end
             nil
           end
 
+          def add_products(*products)
+            @mutex.synchronize { register_products(products) }
+            nil
+          end
+
           def remove_products(*products)
-            @products_mutex.synchronize { products.each { |product| @products.delete(product) } }
+            @mutex.synchronize { products.each { |product| @products.delete(product) } }
             nil
           end
 
           private
 
-          def register(settings, open_feature_component_provider)
+          def register(settings)
             if settings.respond_to?(:appsec) && settings.appsec.enabled
               register_capabilities(Datadog::AppSec::Remote.capabilities)
               register_products(Datadog::AppSec::Remote.products)
@@ -95,32 +111,21 @@ module Datadog
                 end
               end
             end
-
-            if settings.respond_to?(:open_feature) && settings.open_feature.enabled
-              register_capabilities(Datadog::OpenFeature::Remote.capabilities)
-              register_products(Datadog::OpenFeature::Remote.products)
-              register_receivers(
-                Datadog::OpenFeature::Remote.receivers(
-                  @telemetry,
-                  component_provider: open_feature_component_provider,
-                ),
-              )
-            end
           end
 
           def register_capabilities(capabilities)
-            @capabilities.concat(capabilities)
+            capabilities.each { |capability| @capabilities << capability unless @capabilities.include?(capability) }
           end
 
           def register_receivers(receivers)
-            @receivers.concat(receivers)
+            receivers.each { |receiver| @receivers << receiver unless @receivers.include?(receiver) }
           end
 
           def register_products(products)
-            @products.concat(products)
+            products.each { |product| @products << product unless @products.include?(product) }
           end
 
-          def capabilities_to_base64
+          def capabilities_to_base64(capabilities)
             return "" if capabilities.empty?
 
             cap_to_hexs = capabilities.reduce(:|).to_s(16).tap { |s| s.size.odd? && s.prepend("0") }.scan(/\h\h/)

data/lib/datadog/core/remote/client.rb

@@ -16,6 +16,12 @@ module Datadog
 
         attr_reader :transport, :repository, :id, :dispatcher, :settings, :logger
 
+        # Whether the initial reconcile is still pending, to run on the first
+        # sync whose response contains targets.
+        #
+        # @return [Boolean]
+        attr_reader :pending_initial_reconcile
+
         def initialize(transport, capabilities, settings:, logger:, repository: Configuration::Repository.new)
           @transport = transport
           @settings = settings
@@ -25,27 +31,52 @@ module Datadog
           @id = SecureRandom.uuid
           @capabilities = capabilities
           @dispatcher = Dispatcher.new(@capabilities.receivers)
+          @pending_initial_reconcile = true
         end
 
+        # Polls the agent for remote configuration, applies the response to
+        # the repository and dispatches the resulting changes to the receivers.
+        #
+        # The agent sends an empty response when it holds no targets newer
+        # than the targets version the client reported. Until the client
+        # applies a response that carries targets, it reports the initial
+        # targets version, so an empty response then means the agent has not
+        # received targets from the backend. The initial reconcile therefore
+        # waits for a response that carries targets.
+        #
+        # @return [void]
+        # @raise [TransportError] when the agent reports an internal error
+        # @raise [SyncError] when the response cannot be applied to the repository
         def sync
           # TODO: Skip sync if no capabilities are registered
           response = transport.send_config(payload)
 
           if response.ok?
-            process_response(response)
+            changes = process_response(response)
+            if pending_initial_reconcile && !response.empty?
+              dispatcher.reconcile(changes, repository)
+              @pending_initial_reconcile = false
+            end
           elsif response.internal_error?
             raise TransportError, response.to_s
           end
+          nil
         end
 
         private
 
+        # Applies an OK response to the repository and returns the resulting
+        # change set, which is empty when the response is empty.
+        #
+        # @param response [Transport::Config::Response] the response returned by the agent
+        # @return [Configuration::Repository::ChangeSet] the changes applied by this response
+        # @raise [SyncError] when the response cannot be applied to the repository
         def process_response(response)
           # when response is completely empty, do nothing as in: leave as is
           if response.empty?
             logger.debug { "remote: empty response => NOOP" }
 
-            return
+            return Configuration::Repository::ChangeSet.new
           end
 
           begin
@@ -61,7 +92,7 @@ module Datadog
           end
 
           # To make sure steep does not complain
-          return unless paths && targets && contents
+          return Configuration::Repository::ChangeSet.new unless paths && targets && contents
 
           # TODO: sometimes it can strangely be so that paths.empty?
           # TODO: sometimes it can strangely be so that targets.empty?
@@ -69,6 +100,14 @@ module Datadog
           apply_config(paths, targets, contents)
         end
 
+        # Applies the response to the repository in a transaction, dispatches the
+        # resulting changes to the receivers and returns them.
+        #
+        # @param paths [Array<Configuration::Path>] ingress paths from the response
+        # @param targets [Configuration::TargetMap] targets keyed by path
+        # @param contents [Configuration::ContentList] contents from the response
+        # @return [Configuration::Repository::ChangeSet] the changes applied by the transaction
+        # @raise [SyncError] when a path has no matching target or content
         def apply_config(paths, targets, contents)
           changes = repository.transaction do |current, transaction|
             # paths to be removed: previously applied paths minus ingress paths
@@ -121,6 +160,8 @@ module Datadog
           else
             dispatcher.dispatch(changes, repository)
           end
+
+          changes
         end
 
         def payload # standard:disable Metrics/MethodLength