datadog 2.21.0 → 2.23.0

data/lib/datadog/core/telemetry/component.rb

@@ -19,6 +19,8 @@ module Datadog
       #
       # @api private
       class Component
+        ENDPOINT_COLLECTION_MESSAGE_LIMIT = 300
+
         attr_reader :enabled, :logger, :transport, :worker
 
         include Core::Utils::Forking
@@ -107,13 +109,17 @@ module Datadog
           @worker&.enabled = false
         end
 
-        def start(initial_event_is_change = false)
+        def start(initial_event_is_change = false, components:)
           return if !@enabled
 
           initial_event = if initial_event_is_change
-            Event::SynthAppClientConfigurationChange.new(agent_settings: @agent_settings)
+            Event::SynthAppClientConfigurationChange.new(
+              components: components,
+            )
           else
-            Event::AppStarted.new(agent_settings: @agent_settings)
+            Event::AppStarted.new(
+              components: components,
+            )
           end
 
           @worker.start(initial_event)
@@ -165,6 +171,15 @@ module Datadog
           @worker.enqueue(Event::AppClientConfigurationChange.new(changes, 'remote_config'))
         end
 
+        # Report application endpoints
+        def app_endpoints_loaded(endpoints, page_size: ENDPOINT_COLLECTION_MESSAGE_LIMIT)
+          return if !@enabled || forked?
+
+          endpoints.each_slice(page_size).with_index do |endpoints_slice, i|
+            @worker.enqueue(Event::AppEndpointsLoaded.new(endpoints_slice, is_first: i.zero?))
+          end
+        end
+
         # Increments a count metric.
         def inc(namespace, metric_name, value, tags: {}, common: true)
           @metrics_manager.inc(namespace, metric_name, value, tags: tags, common: common)

data/lib/datadog/core/telemetry/emitter.rb

@@ -31,15 +31,15 @@ module Datadog
           seq_id = self.class.sequence.next
           payload = Request.build_payload(event, seq_id, debug: debug?)
           res = @transport.send_telemetry(request_type: event.type, payload: payload)
-          logger.debug { "Telemetry sent for event `#{event.type}` (response code: #{res.code})" }
+          if res.ok?
+            logger.debug { "Telemetry sent for event `#{event.type}`" }
+          else
+            logger.debug { "Failed to send telemetry for event `#{event.type}`: #{res.inspect}" }
+          end
           res
         rescue => e
           logger.debug {
-            "Unable to send telemetry request for event `#{begin
-              event.type
-            rescue
-              "unknown"
-            end}`: #{e}"
+            "Unable to send telemetry request for event `#{event.respond_to?(:type) ? event.type : event.to_s}`: #{e.class}: #{e}"
           }
           Core::Transport::InternalErrorResponse.new(e)
         end

data/lib/datadog/core/telemetry/event/app_endpoints_loaded.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+
+module Datadog
+  module Core
+    module Telemetry
+      module Event
+        # Telemetry event class for sending 'app-endpoints' payload
+        class AppEndpointsLoaded < Base
+          def initialize(endpoints, is_first:)
+            @endpoints = endpoints
+            @is_first = !!is_first
+          end
+
+          def type
+            'app-endpoints'
+          end
+
+          def payload
+            {
+              is_first: @is_first,
+              endpoints: @endpoints
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/event/app_started.rb

@@ -8,8 +8,12 @@ module Datadog
       module Event
         # Telemetry class for the 'app-started' event
         class AppStarted < Base
-          def initialize(agent_settings:)
-            @agent_settings = agent_settings
+          def initialize(components:)
+            # To not hold a reference to the component tree, generate
+            # the event payload here in the constructor.
+            @configuration = configuration(components.settings, components.agent_settings)
+            @install_signature = install_signature(components.settings)
+            @products = products(components)
           end
 
           def type
@@ -18,9 +22,9 @@ module Datadog
 
           def payload
             {
-              products: products,
-              configuration: configuration,
-              install_signature: install_signature,
+              products: @products,
+              configuration: @configuration,
+              install_signature: @install_signature,
               # DEV: Not implemented yet
               # error: error, # Start-up errors
             }
@@ -28,17 +32,18 @@ module Datadog
 
           private
 
-          def products
+          def products(components)
             # @type var products: Hash[Symbol, Hash[Symbol, Hash[Symbol, String | Integer] | bool | nil]]
             products = {
               appsec: {
-                enabled: Datadog::AppSec.enabled?,
+                # TODO take appsec status out of component tree?
+                enabled: components.settings.appsec.enabled,
               },
               profiler: {
-                enabled: Datadog::Profiling.enabled?,
+                enabled: !!components.profiler&.enabled?,
               },
               dynamic_instrumentation: {
-                enabled: defined?(Datadog::DI) && Datadog::DI.respond_to?(:enabled?) && Datadog::DI.enabled?,
+                enabled: !!components.dynamic_instrumentation,
               }
             }
 
@@ -73,12 +78,11 @@ module Datadog
 
           # standard:disable Metrics/AbcSize
           # standard:disable Metrics/MethodLength
-          def configuration
-            config = Datadog.configuration
+          def configuration(settings, agent_settings)
             seq_id = Event.configuration_sequence.next
 
             # tracing.writer_options.buffer_size and tracing.writer_options.flush_interval have the same origin.
-            writer_option_origin = get_telemetry_origin(config, 'tracing.writer_options')
+            writer_option_origin = get_telemetry_origin(settings, 'tracing.writer_options')
 
             list = [
               # Only set using env var as of June 2025
@@ -100,59 +104,59 @@ module Datadog
               ),
 
               # Mix of env var, programmatic and default config, so we use unknown
-              conf_value('DD_AGENT_TRANSPORT', agent_transport, seq_id, 'unknown'), # rubocop:disable CustomCops/EnvStringValidationCop
+              conf_value('DD_AGENT_TRANSPORT', agent_transport(agent_settings), seq_id, 'unknown'), # rubocop:disable CustomCops/EnvStringValidationCop
 
               # writer_options is defined as an option that has a Hash value.
               conf_value(
                 'tracing.writer_options.buffer_size',
-                to_value(config.tracing.writer_options[:buffer_size]),
+                to_value(settings.tracing.writer_options[:buffer_size]),
                 seq_id,
                 writer_option_origin
               ),
               conf_value(
                 'tracing.writer_options.flush_interval',
-                to_value(config.tracing.writer_options[:flush_interval]),
+                to_value(settings.tracing.writer_options[:flush_interval]),
                 seq_id,
                 writer_option_origin
               ),
 
-              conf_value('DD_AGENT_HOST', config.agent.host, seq_id, get_telemetry_origin(config, 'agent.host')),
+              conf_value('DD_AGENT_HOST', settings.agent.host, seq_id, get_telemetry_origin(settings, 'agent.host')),
               conf_value(
                 'DD_TRACE_SAMPLE_RATE',
-                to_value(config.tracing.sampling.default_rate),
+                to_value(settings.tracing.sampling.default_rate),
                 seq_id,
-                get_telemetry_origin(config, 'tracing.sampling.default_rate')
+                get_telemetry_origin(settings, 'tracing.sampling.default_rate')
               ),
               conf_value(
                 'DD_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED',
-                config.tracing.contrib.global_default_service_name.enabled,
+                settings.tracing.contrib.global_default_service_name.enabled,
                 seq_id,
-                get_telemetry_origin(config, 'tracing.contrib.global_default_service_name.enabled')
+                get_telemetry_origin(settings, 'tracing.contrib.global_default_service_name.enabled')
               ),
               conf_value(
                 'DD_TRACE_PEER_SERVICE_DEFAULTS_ENABLED',
-                config.tracing.contrib.peer_service_defaults,
+                settings.tracing.contrib.peer_service_defaults,
                 seq_id,
-                get_telemetry_origin(config, 'tracing.contrib.peer_service_defaults')
+                get_telemetry_origin(settings, 'tracing.contrib.peer_service_defaults')
               ),
               conf_value(
                 'DD_TRACE_DEBUG',
-                config.diagnostics.debug,
+                settings.diagnostics.debug,
                 seq_id,
-                get_telemetry_origin(config, 'diagnostics.debug')
+                get_telemetry_origin(settings, 'diagnostics.debug')
               )
             ]
 
             peer_service_mapping_str = ''
-            unless config.tracing.contrib.peer_service_mapping.empty?
-              peer_service_mapping = config.tracing.contrib.peer_service_mapping
+            unless settings.tracing.contrib.peer_service_mapping.empty?
+              peer_service_mapping = settings.tracing.contrib.peer_service_mapping
               peer_service_mapping_str = peer_service_mapping.map { |key, value| "#{key}:#{value}" }.join(',')
             end
             list << conf_value(
               'DD_TRACE_PEER_SERVICE_MAPPING',
               peer_service_mapping_str,
               seq_id,
-              get_telemetry_origin(config, 'tracing.contrib.peer_service_mapping')
+              get_telemetry_origin(settings, 'tracing.contrib.peer_service_mapping')
             )
 
             # Whitelist of configuration options to send in additional payload object
@@ -160,9 +164,9 @@ module Datadog
               split_option = option_path.split('.')
               list << conf_value(
                 option_path,
-                to_value(config.dig(*split_option)),
+                to_value(settings.dig(*split_option)),
                 seq_id,
-                get_telemetry_origin(config, option_path)
+                get_telemetry_origin(settings, option_path)
               )
             end
 
@@ -181,34 +185,34 @@ module Datadog
             )
 
             # Add some more custom additional payload values here
-            if config.logger.instance
+            if settings.logger.instance
               list << conf_value(
                 'logger.instance',
-                config.logger.instance.class.to_s,
+                settings.logger.instance.class.to_s,
                 seq_id,
-                get_telemetry_origin(config, 'logger.instance')
+                get_telemetry_origin(settings, 'logger.instance')
               )
             end
-            if config.respond_to?('appsec')
+            if settings.respond_to?('appsec')
               list << conf_value(
                 'appsec.enabled',
-                config.dig('appsec', 'enabled'),
+                settings.dig('appsec', 'enabled'),
                 seq_id,
-                get_telemetry_origin(config, 'appsec.enabled')
+                get_telemetry_origin(settings, 'appsec.enabled')
               )
               list << conf_value(
                 'appsec.sca_enabled',
-                config.dig('appsec', 'sca_enabled'),
+                settings.dig('appsec', 'sca_enabled'),
                 seq_id,
-                get_telemetry_origin(config, 'appsec.sca_enabled')
+                get_telemetry_origin(settings, 'appsec.sca_enabled')
               )
             end
-            if config.respond_to?('ci')
+            if settings.respond_to?('ci')
               list << conf_value(
                 'ci.enabled',
-                config.dig('ci', 'enabled'),
+                settings.dig('ci', 'enabled'),
                 seq_id,
-                get_telemetry_origin(config, 'ci.enabled')
+                get_telemetry_origin(settings, 'ci.enabled')
               )
             end
 
@@ -218,8 +222,8 @@ module Datadog
           # standard:enable Metrics/AbcSize
           # standard:enable Metrics/MethodLength
 
-          def agent_transport
-            adapter = @agent_settings.adapter
+          def agent_transport(agent_settings)
+            adapter = agent_settings.adapter
             if adapter == Datadog::Core::Transport::Ext::UnixSocket::ADAPTER
               'UDS'
             else
@@ -260,23 +264,22 @@ module Datadog
             end
           end
 
-          def install_signature
-            config = Datadog.configuration
+          def install_signature(settings)
             {
-              install_id: config.dig('telemetry', 'install_id'),
-              install_type: config.dig('telemetry', 'install_type'),
-              install_time: config.dig('telemetry', 'install_time'),
+              install_id: settings.dig('telemetry', 'install_id'),
+              install_type: settings.dig('telemetry', 'install_type'),
+              install_time: settings.dig('telemetry', 'install_time'),
             }
           end
 
-          def get_telemetry_origin(config, config_path)
+          def get_telemetry_origin(settings, config_path)
             split_option = config_path.split('.')
             option_name = split_option.pop
             return 'unknown' if option_name.nil?
 
             # @type var parent_setting: Core::Configuration::Options
             # @type var option: Core::Configuration::Option
-            parent_setting = config.dig(*split_option)
+            parent_setting = settings.dig(*split_option)
             option = parent_setting.send(:resolve_option, option_name.to_sym)
             option.precedence_set&.origin || 'unknown'
           end

data/lib/datadog/core/telemetry/event/synth_app_client_configuration_change.rb

@@ -33,7 +33,7 @@ module Datadog
 
           def payload
             {
-              configuration: configuration,
+              configuration: @configuration,
             }
           end
         end

data/lib/datadog/core/telemetry/event.rb

@@ -26,6 +26,7 @@ require_relative 'event/base'
 require_relative 'event/app_client_configuration_change'
 require_relative 'event/app_closing'
 require_relative 'event/app_dependencies_loaded'
+require_relative 'event/app_endpoints_loaded'
 require_relative 'event/app_heartbeat'
 require_relative 'event/app_integrations_change'
 require_relative 'event/app_started'

data/lib/datadog/core/telemetry/logger.rb

@@ -14,8 +14,8 @@ module Datadog
       #   read: lib/datadog/core/telemetry/logging.rb
       module Logger
         class << self
-          def report(exception, level: :error, description: nil, pii_safe: false)
-            instance&.report(exception, level: level, description: description, pii_safe: pii_safe)
+          def report(exception, level: :error, description: nil)
+            instance&.report(exception, level: level, description: description)
           end
 
           def error(description)

data/lib/datadog/core/telemetry/logging.rb

@@ -45,17 +45,11 @@ module Datadog
           end
         end
 
-        def report(exception, level: :error, description: nil, pii_safe: false)
+        def report(exception, level: :error, description: nil)
           # Anonymous exceptions to be logged as <Class:0x00007f8b1c0b3b40>
           message = +"#{exception.class.name || exception.class.inspect}" # standard:disable Style/RedundantInterpolation
 
-          exception_message = pii_safe ? exception.message : nil
-
-          if description || exception_message
-            message << ':'
-            message << " #{description}" if description
-            message << " (#{exception.message})" if exception_message
-          end
+          message << ": #{description}" if description
 
           event = Event::Log.new(
             message: message,

data/lib/datadog/core/telemetry/transport/http/telemetry.rb

@@ -3,8 +3,8 @@
 require_relative '../../../transport/http/api/endpoint'
 require_relative '../../../transport/http/api/instance'
 require_relative '../../../transport/http/api/spec'
+require_relative '../../../transport/http/client'
 require_relative '../../../transport/request'
-require_relative 'client'
 
 module Datadog
   module Core
@@ -12,10 +12,11 @@ module Datadog
       module Transport
         module HTTP
           module Telemetry
-            module Client
+            class Client < Core::Transport::HTTP::Client
               def send_telemetry_payload(request)
-                send_request(request) do |api, env| # steep:ignore
-                  api.send_telemetry(env)
+                send_request(request) do |api, env|
+                  # TODO how to make api have the derived type for steep?
+                  api.send_telemetry(env) # steep:ignore
                 end
               end
             end
@@ -83,8 +84,6 @@ module Datadog
               end
             end
           end
-
-          HTTP::Client.include(Telemetry::Client)
         end
       end
     end

data/lib/datadog/core/telemetry/transport/telemetry.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require_relative '../../transport/parcel'
-require_relative 'http/client'
 require_relative 'http/telemetry'
 
 module Datadog
@@ -32,7 +31,7 @@ module Datadog
               @apis = apis
               @logger = logger
 
-              @client = HTTP::Client.new(@apis[default_api], logger: logger)
+              @client = Core::Telemetry::Transport::HTTP::Telemetry::Client.new(@apis[default_api], logger: logger)
             end
 
             def send_telemetry(request_type:, payload:)

data/lib/datadog/core/transport/http/client.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require_relative 'env'
+require_relative '../response'
+
+module Datadog
+  module Core
+    module Transport
+      module HTTP
+        # Routes, encodes, and sends DI data to the trace agent via HTTP.
+        #
+        # @api private
+        class Client
+          attr_reader :api, :logger
+
+          def initialize(api, logger:)
+            @api = api
+            @logger = logger
+          end
+
+          private
+
+          def send_request(request, &block)
+            # Build request into env
+            env = build_env(request)
+
+            # Get responses from API
+            yield(api, env).tap do |response|
+              on_response(response)
+            end
+          rescue => exception
+            on_exception(exception)
+
+            Datadog::Core::Transport::InternalErrorResponse.new(exception)
+          end
+
+          def build_env(request)
+            Datadog::Core::Transport::HTTP::Env.new(request)
+          end
+
+          # Callback that is invoked if a request did not raise an exception
+          # (but did not necessarily complete successfully).
+          #
+          # Override in subclasses.
+          #
+          # Note that the client will return the original response -
+          # the return value of this method is ignored, and response should
+          # not be modified.
+          def on_response(response)
+          end
+
+          # Callback that is invoked if a request failed with an exception.
+          #
+          # Override in subclasses.
+          def on_exception(exception)
+            message = build_exception_message(exception)
+
+            logger.debug(message)
+          end
+
+          def build_exception_message(exception)
+            "Internal error during #{self.class.name} request. Cause: #{exception.class}: #{exception} " \
+              "Location: #{Array(exception.backtrace).first}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/transport/response.rb

@@ -34,7 +34,10 @@ module Datadog
         end
 
         def inspect
-          "#{self.class} ok?:#{ok?} unsupported?:#{unsupported?}, " \
+          maybe_code = if respond_to?(:code)
+            " code:#{code}," # steep:ignore
+          end
+          "#{self.class} ok?:#{ok?},#{maybe_code} unsupported?:#{unsupported?}, " \
             "not_found?:#{not_found?}, client_error?:#{client_error?}, " \
             "server_error?:#{server_error?}, internal_error?:#{internal_error?}, " \
             "payload:#{payload}"

data/lib/datadog/core/utils/array.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Utils
+      # Common array-related utility functions.
+      module Array
+        def self.filter_map(array, &block)
+          if array.respond_to?(:filter_map)
+            # DEV Supported since Ruby 2.7, saves an intermediate object creation
+            array.filter_map(&block)
+          elsif array.is_a?(Enumerator::Lazy)
+            # You would think that .compact would work here, but it does not:
+            # the result of .map could be an Enumerator::Lazy instance which
+            # does not implement #compact on Ruby 2.5/2.6.
+            array.map(&block).reject do |item|
+              item.nil?
+            end
+          else
+            array.each_with_object([]) do |item, memo|
+              new_item = block.call(item)
+              memo.push(new_item) unless new_item.nil?
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/{appsec/api_security → core/utils}/lru_cache.rb

@@ -3,13 +3,13 @@
 require 'forwardable'
 
 module Datadog
-  module AppSec
-    module APISecurity
+  module Core
+    module Utils
       # An LRU (Least Recently Used) cache implementation that relies on the
       # Ruby 1.9+ `Hash` implementation that guarantees insertion order.
       #
       # WARNING: This implementation is NOT thread-safe and should be used
-      #          in a single-threaded context.
+      #          in a single-threaded context or guarded by Mutex.
       class LRUCache
         extend Forwardable
 
@@ -30,25 +30,14 @@ module Datadog
           end
         end
 
-        def store(key, value)
-          return @store[key] = value if @store.delete(key)
-
-          # NOTE: evict the oldest entry if store reached the maximum allowed size
-          @store.shift if @store.size >= @max_size
-          @store[key] = value
-        end
-
-        # NOTE: If the key exists, it's moved to the end of the list and
-        #       if does not, the given block will be executed and the result
-        #       will be stored (which will add it to the end of the list).
-        def fetch_or_store(key)
-          if (entry = @store.delete(key))
-            return @store[key] = entry
+        def []=(key, value)
+          if @store.delete(key)
+            @store[key] = value
+          else
+            # NOTE: evict the oldest entry if store reached the maximum allowed size
+            @store.shift if @store.size >= @max_size
+            @store[key] = value
           end
-
-          # NOTE: evict the oldest entry if store reached the maximum allowed size
-          @store.shift if @store.size >= @max_size
-          @store[key] = yield
         end
       end
     end

data/lib/datadog/core/utils/network.rb

@@ -13,6 +13,7 @@ module Datadog
           true-client-ip
           x-client-ip
           x-forwarded
+          forwarded
           forwarded-for
           x-cluster-client-ip
           fastly-client-ip
@@ -20,6 +21,8 @@ module Datadog
           cf-connecting-ipv6
         ].freeze
 
+        CGNAT_IP_RANGE = IPAddr.new('100.64.0.0/10')
+
         class << self
           # Returns a client IP associated with the request if it was
           #   retrieved successfully.
@@ -73,6 +76,8 @@ module Datadog
               next unless value
 
               ips = value.split(',')
+              ips = process_forwarded_header_values(ips) if name == 'forwarded'
+
               ips.each do |ip|
                 parsed_ip = ip_to_ipaddr(ip.strip)
 
@@ -83,6 +88,22 @@ module Datadog
             nil
           end
 
+          def process_forwarded_header_values(values)
+            values.each_with_object([]) do |value, acc|
+              value.downcase!
+
+              value.split(';').each do |tuple_str|
+                tuple_str.strip!
+                next unless tuple_str.start_with?('for=')
+
+                tuple_str.delete_prefix!('for=')
+                tuple_str.delete!('"')
+
+                acc << tuple_str
+              end
+            end
+          end
+
           # Returns whether the given value is more likely to be an IPv4 than an IPv6 address.
           #
           # This is done by checking if a dot (`'.'`) character appears before a colon (`':'`) in the value.
@@ -112,7 +133,7 @@ module Datadog
           end
 
           def global_ip?(parsed_ip)
-            parsed_ip && !parsed_ip.private? && !parsed_ip.loopback? && !parsed_ip.link_local?
+            parsed_ip && !parsed_ip.private? && !parsed_ip.loopback? && !parsed_ip.link_local? && !CGNAT_IP_RANGE.include?(parsed_ip)
           end
         end
       end