datadog 2.0.0 → 2.2.0

data/lib/datadog/core/configuration/settings.rb

@@ -122,9 +122,18 @@ module Datadog
           # @default `DD_TRACE_DEBUG` environment variable, otherwise `false`
           # @return [Boolean]
           option :debug do |o|
-            o.env Datadog::Core::Configuration::Ext::Diagnostics::ENV_DEBUG_ENABLED
+            o.env [Datadog::Core::Configuration::Ext::Diagnostics::ENV_DEBUG_ENABLED,
+                   Datadog::Core::Configuration::Ext::Diagnostics::ENV_OTEL_LOG_LEVEL]
             o.default false
             o.type :bool
+            o.env_parser do |value|
+              if value
+                value = value.strip.downcase
+                # Debug is enabled when DD_TRACE_DEBUG is true or 1 OR
+                # when OTEL_LOG_LEVEL is set to debug
+                ['true', '1', 'debug'].include?(value)
+              end
+            end
             o.after_set do |enabled|
               # Enable rich debug print statements.
               # We do not need to unnecessarily load 'pp' unless in debugging mode.
@@ -388,6 +397,22 @@ module Datadog
               end
             end
 
+            # The profiler gathers data by sending `SIGPROF` unix signals to Ruby application threads.
+            #
+            # We've discovered that this can trigger a bug in a number of Ruby APIs in the `Dir` class, as
+            # described in https://github.com/DataDog/dd-trace-rb/issues/3450 . This workaround prevents the issue
+            # from happening by monkey patching the affected APIs.
+            #
+            # (In the future, once a fix lands upstream, we'll disable this workaround for Rubies that don't need it)
+            #
+            # @default `DD_PROFILING_DIR_INTERRUPTION_WORKAROUND_ENABLED` environment variable as a boolean,
+            # otherwise `true`
+            option :dir_interruption_workaround_enabled do |o|
+              o.env 'DD_PROFILING_DIR_INTERRUPTION_WORKAROUND_ENABLED'
+              o.type :bool
+              o.default true
+            end
+
             # Configures how much wall-time overhead the profiler targets. The profiler will dynamically adjust the
             # interval between samples it takes so as to try and maintain the property that it spends no longer than
             # this amount of wall-clock time profiling. For example, with the default value of 2%, the profiler will
@@ -465,7 +490,7 @@ module Datadog
           o.type :string, nilable: true
 
           # NOTE: service also gets set as a side effect of tags. See the WORKAROUND note in #initialize for details.
-          o.env Core::Environment::Ext::ENV_SERVICE
+          o.env [Core::Environment::Ext::ENV_SERVICE, Core::Environment::Ext::ENV_OTEL_SERVICE]
           o.default Core::Environment::Ext::FALLBACK_SERVICE_NAME
 
           # There's a few cases where we don't want to use the fallback service name, so this helper allows us to get a
@@ -500,14 +525,13 @@ module Datadog
         # @return [Hash<String,String>]
         option :tags do |o|
           o.type :hash, nilable: true
-          o.env Core::Environment::Ext::ENV_TAGS
+          o.env [Core::Environment::Ext::ENV_TAGS, Core::Environment::Ext::ENV_OTEL_RESOURCE_ATTRIBUTES]
           o.env_parser do |env_value|
             values = if env_value.include?(',')
                        env_value.split(',')
                      else
                        env_value.split(' ') # rubocop:disable Style/RedundantArgument
                      end
-
             values.map! do |v|
               v.gsub!(/\A[\s,]*|[\s,]*\Z/, '')
 
@@ -517,7 +541,23 @@ module Datadog
             values.compact!
             values.each_with_object({}) do |tag, tags|
               key, value = tag.split(':', 2)
-              tags[key] = value if value && !value.empty?
+              if value.nil?
+                # support tags/attributes delimited by the OpenTelemetry separator (`=`)
+                key, value = tag.split('=', 2)
+              end
+              next if value.nil? || value.empty?
+
+              # maps OpenTelemetry semantic attributes to Datadog tags
+              case key.downcase
+              when 'deployment.environment'
+                tags['env'] = value
+              when 'service.version'
+                tags['version'] = value
+              when 'service.name'
+                tags['service'] = value
+              else
+                tags[key] = value
+              end
             end
           end
           o.setter do |new_value, old_value|
@@ -623,6 +663,16 @@ module Datadog
             o.type :bool
           end
 
+          # Enable metrics collection for telemetry. Metrics collection only works when telemetry is enabled and
+          # metrics are enabled.
+          # @default `DD_TELEMETRY_METRICS_ENABLED` environment variable, otherwise `true`.
+          # @return [Boolean]
+          option :metrics_enabled do |o|
+            o.type :bool
+            o.env Core::Telemetry::Ext::ENV_METRICS_ENABLED
+            o.default true
+          end
+
           # The interval in seconds when telemetry must be sent.
           #
           # This method is used internally, for testing purposes only.
@@ -636,6 +686,19 @@ module Datadog
             o.default 60.0
           end
 
+          # The interval in seconds when telemetry metrics are aggregated.
+          # Should be a denominator of `heartbeat_interval_seconds`.
+          #
+          # This method is used internally, for testing purposes only.
+          # @default `DD_TELEMETRY_METRICS_AGGREGATION_INTERVAL` environment variable, otherwise `10`.
+          # @return [Float]
+          # @!visibility private
+          option :metrics_aggregation_interval_seconds do |o|
+            o.type :float
+            o.env Core::Telemetry::Ext::ENV_METRICS_AGGREGATION_INTERVAL
+            o.default 10.0
+          end
+
           # The install id of the application.
           #
           # This method is used internally, by library injection.

data/lib/datadog/core/configuration.rb

@@ -84,23 +84,16 @@ module Datadog
         configuration = self.configuration
         yield(configuration)
 
-        built_components = false
-
-        components = safely_synchronize do |write_components|
+        safely_synchronize do |write_components|
           write_components.call(
             if components?
               replace_components!(configuration, @components)
             else
-              components = build_components(configuration)
-              built_components = true
-              components
+              build_components(configuration)
             end
           )
         end
 
-        # Should only be called the first time components are built
-        components.telemetry.started! if built_components
-
         configuration
       end
 
@@ -200,20 +193,13 @@ module Datadog
         current_components = COMPONENTS_READ_LOCK.synchronize { defined?(@components) && @components }
         return current_components if current_components || !allow_initialization
 
-        built_components = false
-
-        components = safely_synchronize do |write_components|
+        safely_synchronize do |write_components|
           if defined?(@components) && @components
             @components
           else
-            built_components = true
             write_components.call(build_components(configuration))
           end
         end
-
-        # Should only be called the first time components are built
-        components&.telemetry&.started! if built_components
-        components
       end
 
       private

data/lib/datadog/core/deprecations.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    # Contains behavior for handling deprecated functions in the codebase.
+    module Deprecations
+      # Records the occurrence of a deprecated operation in this library.
+      #
+      # Currently, these operations are logged to `Datadog.logger` at `warn` level.
+      #
+      # `disallowed_next_major` adds a message informing that the deprecated operation
+      # won't be allowed in the next major release.
+      #
+      # @yieldreturn [String] a String with the lazily evaluated deprecation message.
+      # @param [Boolean] disallowed_next_major whether this deprecation will be enforced in the next major release.
+      # @param [Object] key A unique key for the deprecation. Only the first message with the same key will be logged.
+      def log_deprecation(disallowed_next_major: true, key: nil)
+        return unless log_deprecation?(key)
+
+        Datadog.logger.warn do
+          message = yield
+          message += ' This will be enforced in the next major release.' if disallowed_next_major
+          message
+        end
+
+        # Track the deprecation being logged.
+        deprecation_logged!(key)
+
+        nil
+      end
+
+      private
+
+      # Determines whether a deprecation message should be logged.
+      #
+      # Internal use only.
+      def log_deprecation?(key)
+        return true if key.nil?
+
+        # Only allow a deprecation to be logged once.
+        !logged_deprecations.key?(key)
+      end
+
+      def deprecation_logged!(key)
+        return if key.nil?
+
+        logged_deprecations[key] += 1
+      end
+
+      # Tracks what deprecation warnings have already been logged
+      #
+      # Internal use only.
+      def logged_deprecations
+        @logged_deprecations ||= Hash.new(0)
+      end
+    end
+  end
+end

data/lib/datadog/core/environment/ext.rb

@@ -18,8 +18,10 @@ module Datadog
         ENV_API_KEY = 'DD_API_KEY'
         ENV_ENVIRONMENT = 'DD_ENV'
         ENV_SERVICE = 'DD_SERVICE'
+        ENV_OTEL_SERVICE = 'OTEL_SERVICE_NAME'
         ENV_SITE = 'DD_SITE'
         ENV_TAGS = 'DD_TAGS'
+        ENV_OTEL_RESOURCE_ATTRIBUTES = 'OTEL_RESOURCE_ATTRIBUTES'
         ENV_VERSION = 'DD_VERSION'
         FALLBACK_SERVICE_NAME =
           begin

data/lib/datadog/core/environment/yjit.rb

@@ -47,6 +47,11 @@ module Datadog
           ::RubyVM::YJIT.runtime_stats[:object_shape_count]
         end
 
+        # Size of memory Rust allocated for metadata
+        def yjit_alloc_size
+          ::RubyVM::YJIT.runtime_stats[:yjit_alloc_size]
+        end
+
         def available?
           defined?(::RubyVM::YJIT) \
             && ::RubyVM::YJIT.enabled? \

data/lib/datadog/core/runtime/ext.rb

@@ -13,6 +13,7 @@ module Datadog
         # @public_api
         module Metrics
           ENV_ENABLED = 'DD_RUNTIME_METRICS_ENABLED'
+          ENV_OTEL_METRICS_EXPORTER = 'OTEL_METRICS_EXPORTER'
 
           METRIC_CLASS_COUNT = 'runtime.ruby.class_count'
           METRIC_GC_PREFIX = 'runtime.ruby.gc'
@@ -29,6 +30,7 @@ module Datadog
           METRIC_YJIT_LIVE_PAGE_COUNT = 'runtime.ruby.yjit.live_page_count'
           METRIC_YJIT_OBJECT_SHAPE_COUNT = 'runtime.ruby.yjit.object_shape_count'
           METRIC_YJIT_OUTLINED_CODE_SIZE = 'runtime.ruby.yjit.outlined_code_size'
+          METRIC_YJIT_YJIT_ALLOC_SIZE = 'runtime.ruby.yjit.yjit_alloc_size'
 
           TAG_SERVICE = 'service'
         end

data/lib/datadog/core/runtime/metrics.rb

@@ -140,6 +140,7 @@ module Datadog
           gauge(metric_name, metric_value) if metric_value
         end
 
+        # rubocop:disable Metrics/MethodLength
         def flush_yjit_stats
           # Only on Ruby >= 3.2
           try_flush do
@@ -176,9 +177,14 @@ module Datadog
                 Core::Runtime::Ext::Metrics::METRIC_YJIT_OUTLINED_CODE_SIZE,
                 Core::Environment::YJIT.outlined_code_size
               )
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_YJIT_YJIT_ALLOC_SIZE,
+                Core::Environment::YJIT.yjit_alloc_size
+              )
             end
           end
         end
+        # rubocop:enable Metrics/MethodLength
       end
     end
   end

data/lib/datadog/core/telemetry/component.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+require_relative 'emitter'
+require_relative 'event'
+require_relative 'metrics_manager'
+require_relative 'worker'
+require_relative '../utils/forking'
+
+module Datadog
+  module Core
+    module Telemetry
+      # Telemetry entrypoint, coordinates sending telemetry events at various points in app lifecycle.
+      class Component
+        attr_reader :enabled
+
+        include Core::Utils::Forking
+
+        # @param enabled [Boolean] Determines whether telemetry events should be sent to the API
+        # @param metrics_enabled [Boolean] Determines whether telemetry metrics should be sent to the API
+        # @param heartbeat_interval_seconds [Float] How frequently heartbeats will be reported, in seconds.
+        # @param metrics_aggregation_interval_seconds [Float] How frequently metrics will be aggregated, in seconds.
+        # @param [Boolean] dependency_collection Whether to send the `app-dependencies-loaded` event
+        def initialize(
+          heartbeat_interval_seconds:,
+          metrics_aggregation_interval_seconds:,
+          dependency_collection:,
+          enabled: true,
+          metrics_enabled: true
+        )
+          @enabled = enabled
+          @stopped = false
+
+          @metrics_manager = MetricsManager.new(
+            enabled: enabled && metrics_enabled,
+            aggregation_interval: metrics_aggregation_interval_seconds
+          )
+
+          @worker = Telemetry::Worker.new(
+            enabled: @enabled,
+            heartbeat_interval_seconds: heartbeat_interval_seconds,
+            metrics_aggregation_interval_seconds: metrics_aggregation_interval_seconds,
+            emitter: Emitter.new,
+            metrics_manager: @metrics_manager,
+            dependency_collection: dependency_collection
+          )
+          @worker.start
+        end
+
+        def disable!
+          @enabled = false
+          @worker.enabled = false
+        end
+
+        def stop!
+          return if @stopped
+
+          @worker.stop(true)
+          @stopped = true
+        end
+
+        def emit_closing!
+          return if !@enabled || forked?
+
+          @worker.enqueue(Event::AppClosing.new)
+        end
+
+        def integrations_change!
+          return if !@enabled || forked?
+
+          @worker.enqueue(Event::AppIntegrationsChange.new)
+        end
+
+        # Report configuration changes caused by Remote Configuration.
+        def client_configuration_change!(changes)
+          return if !@enabled || forked?
+
+          @worker.enqueue(Event::AppClientConfigurationChange.new(changes, 'remote_config'))
+        end
+
+        # Increments a count metric.
+        def inc(namespace, metric_name, value, tags: {}, common: true)
+          @metrics_manager.inc(namespace, metric_name, value, tags: tags, common: common)
+        end
+
+        # Decremenets a count metric.
+        def dec(namespace, metric_name, value, tags: {}, common: true)
+          @metrics_manager.dec(namespace, metric_name, value, tags: tags, common: common)
+        end
+
+        # Tracks gauge metric.
+        def gauge(namespace, metric_name, value, tags: {}, common: true)
+          @metrics_manager.gauge(namespace, metric_name, value, tags: tags, common: common)
+        end
+
+        # Tracks rate metric.
+        def rate(namespace, metric_name, value, tags: {}, common: true)
+          @metrics_manager.rate(namespace, metric_name, value, tags: tags, common: common)
+        end
+
+        # Tracks distribution metric.
+        def distribution(namespace, metric_name, value, tags: {}, common: true)
+          @metrics_manager.distribution(namespace, metric_name, value, tags: tags, common: common)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/event.rb

@@ -3,7 +3,16 @@
 module Datadog
   module Core
     module Telemetry
+      # Collection of telemetry events
       class Event
+        extend Core::Utils::Forking
+
+        # returns sequence that increments every time the configuration changes
+        def self.configuration_sequence
+          after_fork! { @sequence = Datadog::Core::Utils::Sequence.new(1) }
+          @sequence ||= Datadog::Core::Utils::Sequence.new(1)
+        end
+
         # Base class for all Telemetry V2 events.
         class Base
           # The type of the event.
@@ -12,8 +21,7 @@ module Datadog
           def type; end
 
           # The JSON payload for the event.
-          # @param seq_id [Integer] The sequence ID for the event.
-          def payload(seq_id)
+          def payload
             {}
           end
         end
@@ -24,8 +32,7 @@ module Datadog
             'app-started'
           end
 
-          def payload(seq_id)
-            @seq_id = seq_id
+          def payload
             {
               products: products,
               configuration: configuration,
@@ -38,6 +45,7 @@ module Datadog
           private
 
           def products
+            # @type var products: Hash[Symbol, Hash[Symbol, Object]]
             products = {
               appsec: {
                 enabled: Datadog::AppSec.enabled?,
@@ -79,16 +87,19 @@ module Datadog
           ].freeze
 
           # rubocop:disable Metrics/AbcSize
+          # rubocop:disable Metrics/MethodLength
           def configuration
             config = Datadog.configuration
+            seq_id = Event.configuration_sequence.next
 
             list = [
-              conf_value('DD_AGENT_HOST', config.agent.host),
-              conf_value('DD_AGENT_TRANSPORT', agent_transport(config)),
-              conf_value('DD_TRACE_SAMPLE_RATE', to_value(config.tracing.sampling.default_rate)),
+              conf_value('DD_AGENT_HOST', config.agent.host, seq_id),
+              conf_value('DD_AGENT_TRANSPORT', agent_transport(config), seq_id),
+              conf_value('DD_TRACE_SAMPLE_RATE', to_value(config.tracing.sampling.default_rate), seq_id),
               conf_value(
                 'DD_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED',
-                config.tracing.contrib.global_default_service_name.enabled
+                config.tracing.contrib.global_default_service_name.enabled,
+                seq_id
               ),
             ]
 
@@ -97,29 +108,45 @@ module Datadog
               peer_service_mapping = config.tracing.contrib.peer_service_mapping
               peer_service_mapping_str = peer_service_mapping.map { |key, value| "#{key}:#{value}" }.join(',')
             end
-            list << conf_value('DD_TRACE_PEER_SERVICE_MAPPING', peer_service_mapping_str)
+            list << conf_value('DD_TRACE_PEER_SERVICE_MAPPING', peer_service_mapping_str, seq_id)
 
             # Whitelist of configuration options to send in additional payload object
             TARGET_OPTIONS.each do |option|
               split_option = option.split('.')
-              list << conf_value(option, to_value(config.dig(*split_option)))
+              list << conf_value(option, to_value(config.dig(*split_option)), seq_id)
             end
 
             # Add some more custom additional payload values here
             list.push(
-              conf_value('tracing.auto_instrument.enabled', !defined?(Datadog::AutoInstrument::LOADED).nil?),
-              conf_value('tracing.writer_options.buffer_size', to_value(config.tracing.writer_options[:buffer_size])),
-              conf_value('tracing.writer_options.flush_interval', to_value(config.tracing.writer_options[:flush_interval])),
-              conf_value('tracing.opentelemetry.enabled', !defined?(Datadog::OpenTelemetry::LOADED).nil?),
+              conf_value('tracing.auto_instrument.enabled', !defined?(Datadog::AutoInstrument::LOADED).nil?, seq_id),
+              conf_value(
+                'tracing.writer_options.buffer_size',
+                to_value(config.tracing.writer_options[:buffer_size]),
+                seq_id
+              ),
+              conf_value(
+                'tracing.writer_options.flush_interval',
+                to_value(config.tracing.writer_options[:flush_interval]),
+                seq_id
+              ),
+              conf_value(
+                'tracing.opentelemetry.enabled',
+                !defined?(Datadog::OpenTelemetry::LOADED).nil?,
+                seq_id
+              ),
             )
-            list << conf_value('logger.instance', config.logger.instance.class.to_s) if config.logger.instance
-            list << conf_value('appsec.enabled', config.dig('appsec', 'enabled')) if config.respond_to?('appsec')
-            list << conf_value('ci.enabled', config.dig('ci', 'enabled')) if config.respond_to?('ci')
+            list << conf_value('logger.instance', config.logger.instance.class.to_s, seq_id) if config.logger.instance
+            if config.respond_to?('appsec')
+              list << conf_value('appsec.enabled', config.dig('appsec', 'enabled'), seq_id)
+              list << conf_value('appsec.sca_enabled', config.dig('appsec', 'sca_enabled'), seq_id)
+            end
+            list << conf_value('ci.enabled', config.dig('ci', 'enabled'), seq_id) if config.respond_to?('ci')
 
             list.reject! { |entry| entry[:value].nil? }
             list
           end
           # rubocop:enable Metrics/AbcSize
+          # rubocop:enable Metrics/MethodLength
 
           def agent_transport(config)
             adapter = Core::Configuration::AgentSettingsResolver.call(config).adapter
@@ -130,12 +157,12 @@ module Datadog
             end
           end
 
-          def conf_value(name, value, origin = 'code')
+          def conf_value(name, value, seq_id, origin = 'code')
             {
               name: name,
               value: value,
               origin: origin,
-              seq_id: @seq_id,
+              seq_id: seq_id,
             }
           end
 
@@ -165,7 +192,7 @@ module Datadog
             'app-dependencies-loaded'
           end
 
-          def payload(seq_id)
+          def payload
             { dependencies: dependencies }
           end
 
@@ -188,7 +215,7 @@ module Datadog
             'app-integrations-change'
           end
 
-          def payload(seq_id)
+          def payload
             { integrations: integrations }
           end
 
@@ -241,16 +268,33 @@ module Datadog
             @origin = origin
           end
 
-          def payload(seq_id)
-            {
-              configuration: @changes.map do |name, value|
-                {
-                  name: name,
-                  value: value,
-                  origin: @origin,
-                }
-              end
-            }
+          def payload
+            { configuration: configuration }
+          end
+
+          def configuration
+            config = Datadog.configuration
+            seq_id = Event.configuration_sequence.next
+
+            res = @changes.map do |name, value|
+              {
+                name: name,
+                value: value,
+                origin: @origin,
+                seq_id: seq_id,
+              }
+            end
+
+            unless config.dig('appsec', 'sca_enabled').nil?
+              res << {
+                name: 'appsec.sca_enabled',
+                value: config.appsec.sca_enabled,
+                origin: 'code',
+                seq_id: seq_id,
+              }
+            end
+
+            res
           end
         end
 
@@ -267,6 +311,55 @@ module Datadog
             'app-closing'
           end
         end
+
+        # Telemetry class for the 'generate-metrics' event
+        class GenerateMetrics < Base
+          def type
+            'generate-metrics'
+          end
+
+          def initialize(namespace, metric_series)
+            super()
+            @namespace = namespace
+            @metric_series = metric_series
+          end
+
+          def payload
+            {
+              namespace: @namespace,
+              series: @metric_series.map(&:to_h)
+            }
+          end
+        end
+
+        # Telemetry class for the 'distributions' event
+        class Distributions < GenerateMetrics
+          def type
+            'distributions'
+          end
+        end
+
+        # Telemetry class for the 'message-batch' event
+        class MessageBatch
+          attr_reader :events
+
+          def type
+            'message-batch'
+          end
+
+          def initialize(events)
+            @events = events
+          end
+
+          def payload
+            @events.map do |event|
+              {
+                request_type: event.type,
+                payload: event.payload,
+              }
+            end
+          end
+        end
       end
     end
   end

data/lib/datadog/core/telemetry/ext.rb

@@ -5,7 +5,9 @@ module Datadog
     module Telemetry
       module Ext
         ENV_ENABLED = 'DD_INSTRUMENTATION_TELEMETRY_ENABLED'
+        ENV_METRICS_ENABLED = 'DD_TELEMETRY_METRICS_ENABLED'
         ENV_HEARTBEAT_INTERVAL = 'DD_TELEMETRY_HEARTBEAT_INTERVAL'
+        ENV_METRICS_AGGREGATION_INTERVAL = 'DD_TELEMETRY_METRICS_AGGREGATION_INTERVAL'
         ENV_DEPENDENCY_COLLECTION = 'DD_TELEMETRY_DEPENDENCY_COLLECTION_ENABLED'
         ENV_INSTALL_ID = 'DD_INSTRUMENTATION_INSTALL_ID'
         ENV_INSTALL_TYPE = 'DD_INSTRUMENTATION_INSTALL_TYPE'

data/lib/datadog/core/telemetry/http/adapters/net.rb

@@ -15,7 +15,7 @@ module Datadog
               :timeout,
               :ssl
 
-            DEFAULT_TIMEOUT = 30
+            DEFAULT_TIMEOUT = 2
 
             def initialize(hostname:, port: nil, timeout: DEFAULT_TIMEOUT, ssl: true)
               @hostname = hostname