datadog 2.1.0 → 2.3.0

data/lib/datadog/core/configuration/settings.rb

@@ -301,6 +301,16 @@ module Datadog
               o.default true
             end
 
+            # Can be used to enable/disable the Datadog::Profiling.allocation_count feature.
+            #
+            # Requires allocation profiling to be enabled.
+            #
+            # @default false
+            option :allocation_counting_enabled do |o|
+              o.type :bool
+              o.default false
+            end
+
             # Can be used to enable/disable the collection of heap profiles.
             #
             # This feature is alpha and disabled by default
@@ -397,6 +407,22 @@ module Datadog
               end
             end
 
+            # The profiler gathers data by sending `SIGPROF` unix signals to Ruby application threads.
+            #
+            # We've discovered that this can trigger a bug in a number of Ruby APIs in the `Dir` class, as
+            # described in https://github.com/DataDog/dd-trace-rb/issues/3450 . This workaround prevents the issue
+            # from happening by monkey patching the affected APIs.
+            #
+            # (In the future, once a fix lands upstream, we'll disable this workaround for Rubies that don't need it)
+            #
+            # @default `DD_PROFILING_DIR_INTERRUPTION_WORKAROUND_ENABLED` environment variable as a boolean,
+            # otherwise `true`
+            option :dir_interruption_workaround_enabled do |o|
+              o.env 'DD_PROFILING_DIR_INTERRUPTION_WORKAROUND_ENABLED'
+              o.type :bool
+              o.default true
+            end
+
             # Configures how much wall-time overhead the profiler targets. The profiler will dynamically adjust the
             # interval between samples it takes so as to try and maintain the property that it spends no longer than
             # this amount of wall-clock time profiling. For example, with the default value of 2%, the profiler will
@@ -425,14 +451,16 @@ module Datadog
               o.default 60
             end
 
-            # Enables reporting of information when the Ruby VM crashes.
-            #
-            # @default `DD_PROFILING_EXPERIMENTAL_CRASH_TRACKING_ENABLED` environment variable as a boolean,
-            # otherwise `false`
+            # DEV-3.0: Remove `experimental_crash_tracking_enabled` option
             option :experimental_crash_tracking_enabled do |o|
-              o.type :bool
-              o.env 'DD_PROFILING_EXPERIMENTAL_CRASH_TRACKING_ENABLED'
-              o.default false
+              o.after_set do |_, _, precedence|
+                unless precedence == Datadog::Core::Configuration::Option::Precedence::DEFAULT
+                  Core.log_deprecation(key: :experimental_crash_tracking_enabled) do
+                    'The profiling.advanced.experimental_crash_tracking_enabled setting has been deprecated for removal '\
+                    'and no longer does anything. Please remove it from your Datadog.configure block.'
+                  end
+                end
+              end
             end
           end
 
@@ -647,6 +675,34 @@ module Datadog
             o.type :bool
           end
 
+          # Enable agentless mode for telemetry: submit telemetry events directly to the intake without Datadog Agent.
+          #
+          # @return [Boolean]
+          # @!visibility private
+          option :agentless_enabled do |o|
+            o.type :bool
+            o.default false
+          end
+
+          # Overrides agentless telemetry URL. To be used internally for testing purposes only.
+          #
+          # @return [String]
+          # @!visibility private
+          option :agentless_url_override do |o|
+            o.type :string, nilable: true
+            o.env Core::Telemetry::Ext::ENV_AGENTLESS_URL_OVERRIDE
+          end
+
+          # Enable metrics collection for telemetry. Metrics collection only works when telemetry is enabled and
+          # metrics are enabled.
+          # @default `DD_TELEMETRY_METRICS_ENABLED` environment variable, otherwise `true`.
+          # @return [Boolean]
+          option :metrics_enabled do |o|
+            o.type :bool
+            o.env Core::Telemetry::Ext::ENV_METRICS_ENABLED
+            o.default true
+          end
+
           # The interval in seconds when telemetry must be sent.
           #
           # This method is used internally, for testing purposes only.
@@ -660,6 +716,19 @@ module Datadog
             o.default 60.0
           end
 
+          # The interval in seconds when telemetry metrics are aggregated.
+          # Should be a denominator of `heartbeat_interval_seconds`.
+          #
+          # This method is used internally, for testing purposes only.
+          # @default `DD_TELEMETRY_METRICS_AGGREGATION_INTERVAL` environment variable, otherwise `10`.
+          # @return [Float]
+          # @!visibility private
+          option :metrics_aggregation_interval_seconds do |o|
+            o.type :float
+            o.env Core::Telemetry::Ext::ENV_METRICS_AGGREGATION_INTERVAL
+            o.default 10.0
+          end
+
           # The install id of the application.
           #
           # This method is used internally, by library injection.
@@ -695,6 +764,14 @@ module Datadog
             o.type :string, nilable: true
             o.env Core::Telemetry::Ext::ENV_INSTALL_TIME
           end
+
+          # Telemetry shutdown timeout in seconds
+          #
+          # @!visibility private
+          option :shutdown_timeout_seconds do |o|
+            o.type :float
+            o.default 1.0
+          end
         end
 
         # Remote configuration
@@ -755,6 +832,15 @@ module Datadog
           option :service
         end
 
+        settings :crashtracking do
+          # Enables reporting of information when Ruby VM crashes.
+          option :enabled do |o|
+            o.type :bool
+            o.default true
+            o.env 'DD_CRASHTRACKING_ENABLED'
+          end
+        end
+
         # TODO: Tracing should manage its own settings.
         #       Keep this extension here for now to keep things working.
         extend Datadog::Tracing::Configuration::Settings

data/lib/datadog/core/configuration.rb

@@ -84,23 +84,16 @@ module Datadog
         configuration = self.configuration
         yield(configuration)
 
-        built_components = false
-
-        components = safely_synchronize do |write_components|
+        safely_synchronize do |write_components|
           write_components.call(
             if components?
               replace_components!(configuration, @components)
             else
-              components = build_components(configuration)
-              built_components = true
-              components
+              build_components(configuration)
             end
           )
         end
 
-        # Should only be called the first time components are built
-        components.telemetry.started! if built_components
-
         configuration
       end
 
@@ -200,20 +193,13 @@ module Datadog
         current_components = COMPONENTS_READ_LOCK.synchronize { defined?(@components) && @components }
         return current_components if current_components || !allow_initialization
 
-        built_components = false
-
-        components = safely_synchronize do |write_components|
+        safely_synchronize do |write_components|
           if defined?(@components) && @components
             @components
           else
-            built_components = true
             write_components.call(build_components(configuration))
           end
         end
-
-        # Should only be called the first time components are built
-        components&.telemetry&.started! if built_components
-        components
       end
 
       private

data/lib/datadog/core/crashtracking/agent_base_url.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require_relative '../configuration/ext'
+
+module Datadog
+  module Core
+    module Crashtracking
+      # This module provides a method to resolve the base URL of the agent
+      module AgentBaseUrl
+        def self.resolve(agent_settings)
+          case agent_settings.adapter
+          when Datadog::Core::Configuration::Ext::Agent::HTTP::ADAPTER
+            "#{agent_settings.ssl ? 'https' : 'http'}://#{agent_settings.hostname}:#{agent_settings.port}/"
+          when Datadog::Core::Configuration::Ext::Agent::UnixSocket::ADAPTER
+            "unix://#{agent_settings.uds_path}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/crashtracking/component.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require 'libdatadog'
+
+require_relative 'tag_builder'
+require_relative 'agent_base_url'
+require_relative '../utils/only_once'
+require_relative '../utils/at_fork_monkey_patch'
+
+module Datadog
+  module Core
+    module Crashtracking
+      # Used to report Ruby VM crashes.
+      #
+      # NOTE: The crashtracker native state is a singleton;
+      # so even if you create multiple instances of `Crashtracking::Component` and start them,
+      # it only works as "last writer wins". Same for stop -- there's only one state, so calling stop
+      # on it will stop the crash tracker, regardless of which instance started it.
+      #
+      # Methods prefixed with _native_ are implemented in `crashtracker.c`
+      class Component
+        LIBDATADOG_API_FAILURE =
+          begin
+            require "libdatadog_api.#{RUBY_VERSION[/\d+.\d+/]}_#{RUBY_PLATFORM}"
+            nil
+          rescue LoadError => e
+            e.message
+          end
+
+        ONLY_ONCE = Core::Utils::OnlyOnce.new
+
+        def self.build(settings, agent_settings, logger:)
+          tags = TagBuilder.call(settings)
+          agent_base_url = AgentBaseUrl.resolve(agent_settings)
+          logger.warn('Missing agent base URL; cannot enable crash tracking') unless agent_base_url
+
+          ld_library_path = ::Libdatadog.ld_library_path
+          logger.warn('Missing ld_library_path; cannot enable crash tracking') unless ld_library_path
+
+          path_to_crashtracking_receiver_binary = ::Libdatadog.path_to_crashtracking_receiver_binary
+          unless path_to_crashtracking_receiver_binary
+            logger.warn('Missing path_to_crashtracking_receiver_binary; cannot enable crash tracking')
+          end
+
+          return unless agent_base_url
+          return unless ld_library_path
+          return unless path_to_crashtracking_receiver_binary
+
+          new(
+            tags: tags,
+            agent_base_url: agent_base_url,
+            ld_library_path: ld_library_path,
+            path_to_crashtracking_receiver_binary: path_to_crashtracking_receiver_binary,
+            logger: logger
+          ).tap(&:start)
+        end
+
+        def initialize(tags:, agent_base_url:, ld_library_path:, path_to_crashtracking_receiver_binary:, logger:)
+          @tags = tags
+          @agent_base_url = agent_base_url
+          @ld_library_path = ld_library_path
+          @path_to_crashtracking_receiver_binary = path_to_crashtracking_receiver_binary
+          @logger = logger
+        end
+
+        def start
+          Utils::AtForkMonkeyPatch.apply!
+
+          start_or_update_on_fork(action: :start)
+          ONLY_ONCE.run do
+            Utils::AtForkMonkeyPatch.at_fork(:child) do
+              # Must NOT reference `self` here, as only the first instance will
+              # be captured by the ONLY_ONCE and we want to pick the latest active one
+              # (which may have different tags or agent config)
+              Datadog.send(:components).crashtracker&.update_on_fork
+            end
+          end
+        end
+
+        def update_on_fork
+          start_or_update_on_fork(action: :update_on_fork)
+        end
+
+        def stop
+          self.class._native_stop
+          logger.debug('Crash tracking stopped successfully')
+        rescue => e
+          logger.error("Failed to stop crash tracking: #{e.message}")
+        end
+
+        private
+
+        attr_reader :tags, :agent_base_url, :ld_library_path, :path_to_crashtracking_receiver_binary, :logger
+
+        def start_or_update_on_fork(action:)
+          self.class._native_start_or_update_on_fork(
+            action: action,
+            exporter_configuration: [:agent, agent_base_url],
+            path_to_crashtracking_receiver_binary: path_to_crashtracking_receiver_binary,
+            ld_library_path: ld_library_path,
+            tags_as_array: tags.to_a,
+            upload_timeout_seconds: 1
+          )
+          logger.debug("Crash tracking #{action} successfully")
+        rescue => e
+          logger.error("Failed to #{action} crash tracking: #{e.message}")
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/crashtracking/tag_builder.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require_relative '../utils'
+require_relative '../environment/socket'
+require_relative '../environment/identity'
+require_relative '../environment/git'
+
+module Datadog
+  module Core
+    module Crashtracking
+      # This module builds a hash of tags
+      module TagBuilder
+        def self.call(settings)
+          hash = {
+            'host' => Environment::Socket.hostname,
+            'process_id' => Process.pid.to_s,
+            'runtime_engine' => Environment::Identity.lang_engine,
+            'runtime-id' => Environment::Identity.id,
+            'runtime_platform' => Environment::Identity.lang_platform,
+            'runtime_version' => Environment::Identity.lang_version,
+            'env' => settings.env,
+            'service' => settings.service,
+            'version' => settings.version,
+            'git.repository_url' => Environment::Git.git_repository_url,
+            'git.commit.sha' => Environment::Git.git_commit_sha,
+            'is_crash' => 'true',
+            'language' => 'ruby',
+            'library_version' => Core::Environment::Identity.gem_datadog_version,
+          }.compact
+
+          # Make sure everything is an utf-8 string, to avoid encoding issues in downstream
+          settings.tags.merge(hash).each_with_object({}) do |(key, value), h|
+            h[Utils.utf8_encode(key)] = Utils.utf8_encode(value)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/deprecations.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    # Contains behavior for handling deprecated functions in the codebase.
+    module Deprecations
+      # Records the occurrence of a deprecated operation in this library.
+      #
+      # Currently, these operations are logged to `Datadog.logger` at `warn` level.
+      #
+      # `disallowed_next_major` adds a message informing that the deprecated operation
+      # won't be allowed in the next major release.
+      #
+      # @yieldreturn [String] a String with the lazily evaluated deprecation message.
+      # @param [Boolean] disallowed_next_major whether this deprecation will be enforced in the next major release.
+      # @param [Object] key A unique key for the deprecation. Only the first message with the same key will be logged.
+      def log_deprecation(disallowed_next_major: true, key: nil)
+        return unless log_deprecation?(key)
+
+        Datadog.logger.warn do
+          message = yield
+          message += ' This will be enforced in the next major release.' if disallowed_next_major
+          message
+        end
+
+        # Track the deprecation being logged.
+        deprecation_logged!(key)
+
+        nil
+      end
+
+      private
+
+      # Determines whether a deprecation message should be logged.
+      #
+      # Internal use only.
+      def log_deprecation?(key)
+        return true if key.nil?
+
+        # Only allow a deprecation to be logged once.
+        !logged_deprecations.key?(key)
+      end
+
+      def deprecation_logged!(key)
+        return if key.nil?
+
+        logged_deprecations[key] += 1
+      end
+
+      # Tracks what deprecation warnings have already been logged
+      #
+      # Internal use only.
+      def logged_deprecations
+        @logged_deprecations ||= Hash.new(0)
+      end
+    end
+  end
+end

data/lib/datadog/core/diagnostics/environment_logger.rb

@@ -8,11 +8,17 @@ module Datadog
   module Core
     module Diagnostics
       # Base class for EnvironmentLoggers - should allow for easy reporting by users to Datadog support.
+      #
+      # The EnvironmentLogger should not pollute the logs in a development environment.
       module EnvironmentLogging
         def log_configuration!(prefix, data)
           logger.info("DATADOG CONFIGURATION - #{prefix} - #{data}")
         end
 
+        def log_debug!(prefix, data)
+          logger.debug("DATADOG CONFIGURATION - #{prefix} - #{data}")
+        end
+
         def log_error!(prefix, type, error)
           logger.warn("DATADOG ERROR - #{prefix} - #{type}: #{error}")
         end
@@ -27,21 +33,12 @@ module Datadog
         def log?
           startup_logs_enabled = Datadog.configuration.diagnostics.startup_logs.enabled
           if startup_logs_enabled.nil?
-            !repl? && !rspec? # Suppress logs if we are running in a REPL or rspec
+            # Do not pollute the logs in a development environment.
+            !Datadog::Core::Environment::Execution.development?
           else
             startup_logs_enabled
           end
         end
-
-        REPL_PROGRAM_NAMES = %w[irb pry].freeze
-
-        def repl?
-          REPL_PROGRAM_NAMES.include?($PROGRAM_NAME)
-        end
-
-        def rspec?
-          $PROGRAM_NAME.end_with?('rspec')
-        end
       end
 
       # Collects and logs Core diagnostic information

data/lib/datadog/core/environment/yjit.rb

@@ -47,6 +47,11 @@ module Datadog
           ::RubyVM::YJIT.runtime_stats[:object_shape_count]
         end
 
+        # Size of memory Rust allocated for metadata
+        def yjit_alloc_size
+          ::RubyVM::YJIT.runtime_stats[:yjit_alloc_size]
+        end
+
         def available?
           defined?(::RubyVM::YJIT) \
             && ::RubyVM::YJIT.enabled? \

data/lib/datadog/core/runtime/ext.rb

@@ -30,6 +30,7 @@ module Datadog
           METRIC_YJIT_LIVE_PAGE_COUNT = 'runtime.ruby.yjit.live_page_count'
           METRIC_YJIT_OBJECT_SHAPE_COUNT = 'runtime.ruby.yjit.object_shape_count'
           METRIC_YJIT_OUTLINED_CODE_SIZE = 'runtime.ruby.yjit.outlined_code_size'
+          METRIC_YJIT_YJIT_ALLOC_SIZE = 'runtime.ruby.yjit.yjit_alloc_size'
 
           TAG_SERVICE = 'service'
         end

data/lib/datadog/core/runtime/metrics.rb

@@ -140,6 +140,7 @@ module Datadog
           gauge(metric_name, metric_value) if metric_value
         end
 
+        # rubocop:disable Metrics/MethodLength
         def flush_yjit_stats
           # Only on Ruby >= 3.2
           try_flush do
@@ -176,9 +177,14 @@ module Datadog
                 Core::Runtime::Ext::Metrics::METRIC_YJIT_OUTLINED_CODE_SIZE,
                 Core::Environment::YJIT.outlined_code_size
               )
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_YJIT_YJIT_ALLOC_SIZE,
+                Core::Environment::YJIT.yjit_alloc_size
+              )
             end
           end
         end
+        # rubocop:enable Metrics/MethodLength
       end
     end
   end

data/lib/datadog/core/telemetry/component.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+require_relative 'emitter'
+require_relative 'event'
+require_relative 'http/transport'
+require_relative 'metrics_manager'
+require_relative 'worker'
+
+require_relative '../configuration/ext'
+require_relative '../utils/forking'
+
+module Datadog
+  module Core
+    module Telemetry
+      # Telemetry entrypoint, coordinates sending telemetry events at various points in app lifecycle.
+      class Component
+        attr_reader :enabled
+
+        include Core::Utils::Forking
+
+        def self.build(settings, agent_settings, logger)
+          enabled = settings.telemetry.enabled
+          agentless_enabled = settings.telemetry.agentless_enabled
+
+          if !agentless_enabled && agent_settings.adapter != Datadog::Core::Configuration::Ext::Agent::HTTP::ADAPTER
+            enabled = false
+            logger.debug { "Telemetry disabled. Agent network adapter not supported: #{agent_settings.adapter}" }
+          end
+
+          if agentless_enabled && settings.api_key.nil?
+            enabled = false
+            logger.debug { 'Telemetry disabled. Agentless telemetry requires an DD_API_KEY variable to be set.' }
+          end
+
+          transport = if agentless_enabled
+                        Datadog::Core::Telemetry::Http::Transport.build_agentless_transport(
+                          api_key: settings.api_key,
+                          dd_site: settings.site,
+                          url_override: settings.telemetry.agentless_url_override
+                        )
+                      else
+                        Datadog::Core::Telemetry::Http::Transport.build_agent_transport(agent_settings)
+                      end
+
+          Telemetry::Component.new(
+            http_transport: transport,
+            enabled: enabled,
+            metrics_enabled: enabled && settings.telemetry.metrics_enabled,
+            heartbeat_interval_seconds: settings.telemetry.heartbeat_interval_seconds,
+            metrics_aggregation_interval_seconds: settings.telemetry.metrics_aggregation_interval_seconds,
+            dependency_collection: settings.telemetry.dependency_collection,
+            shutdown_timeout_seconds: settings.telemetry.shutdown_timeout_seconds,
+          )
+        end
+
+        # @param enabled [Boolean] Determines whether telemetry events should be sent to the API
+        # @param metrics_enabled [Boolean] Determines whether telemetry metrics should be sent to the API
+        # @param heartbeat_interval_seconds [Float] How frequently heartbeats will be reported, in seconds.
+        # @param metrics_aggregation_interval_seconds [Float] How frequently metrics will be aggregated, in seconds.
+        # @param [Boolean] dependency_collection Whether to send the `app-dependencies-loaded` event
+        def initialize(
+          heartbeat_interval_seconds:,
+          metrics_aggregation_interval_seconds:,
+          dependency_collection:,
+          http_transport:,
+          shutdown_timeout_seconds:,
+          enabled: true,
+          metrics_enabled: true
+        )
+          @enabled = enabled
+          @stopped = false
+
+          @metrics_manager = MetricsManager.new(
+            enabled: enabled && metrics_enabled,
+            aggregation_interval: metrics_aggregation_interval_seconds
+          )
+
+          @worker = Telemetry::Worker.new(
+            enabled: @enabled,
+            heartbeat_interval_seconds: heartbeat_interval_seconds,
+            metrics_aggregation_interval_seconds: metrics_aggregation_interval_seconds,
+            emitter: Emitter.new(http_transport: http_transport),
+            metrics_manager: @metrics_manager,
+            dependency_collection: dependency_collection,
+            shutdown_timeout: shutdown_timeout_seconds
+          )
+          @worker.start
+        end
+
+        def disable!
+          @enabled = false
+          @worker.enabled = false
+        end
+
+        def stop!
+          return if @stopped
+
+          @worker.stop(true)
+          @stopped = true
+        end
+
+        def emit_closing!
+          return if !@enabled || forked?
+
+          @worker.enqueue(Event::AppClosing.new)
+        end
+
+        def integrations_change!
+          return if !@enabled || forked?
+
+          @worker.enqueue(Event::AppIntegrationsChange.new)
+        end
+
+        def log!(event)
+          return unless @enabled || forked?
+
+          @worker.enqueue(event)
+        end
+
+        # Report configuration changes caused by Remote Configuration.
+        def client_configuration_change!(changes)
+          return if !@enabled || forked?
+
+          @worker.enqueue(Event::AppClientConfigurationChange.new(changes, 'remote_config'))
+        end
+
+        # Increments a count metric.
+        def inc(namespace, metric_name, value, tags: {}, common: true)
+          @metrics_manager.inc(namespace, metric_name, value, tags: tags, common: common)
+        end
+
+        # Decremenets a count metric.
+        def dec(namespace, metric_name, value, tags: {}, common: true)
+          @metrics_manager.dec(namespace, metric_name, value, tags: tags, common: common)
+        end
+
+        # Tracks gauge metric.
+        def gauge(namespace, metric_name, value, tags: {}, common: true)
+          @metrics_manager.gauge(namespace, metric_name, value, tags: tags, common: common)
+        end
+
+        # Tracks rate metric.
+        def rate(namespace, metric_name, value, tags: {}, common: true)
+          @metrics_manager.rate(namespace, metric_name, value, tags: tags, common: common)
+        end
+
+        # Tracks distribution metric.
+        def distribution(namespace, metric_name, value, tags: {}, common: true)
+          @metrics_manager.distribution(namespace, metric_name, value, tags: tags, common: common)
+        end
+      end
+    end
+  end
+end