datadog 2.1.0 → 2.2.0

data/lib/datadog/core/telemetry/worker.rb

@@ -0,0 +1,173 @@
+# frozen_string_literal: true
+
+require_relative 'event'
+
+require_relative '../utils/only_once_successful'
+require_relative '../workers/polling'
+require_relative '../workers/queue'
+
+module Datadog
+  module Core
+    module Telemetry
+      # Accumulates events and sends them to the API at a regular interval, including heartbeat event.
+      class Worker
+        include Core::Workers::Queue
+        include Core::Workers::Polling
+
+        DEFAULT_BUFFER_MAX_SIZE = 1000
+        APP_STARTED_EVENT_RETRIES = 10
+
+        TELEMETRY_STARTED_ONCE = Utils::OnlyOnceSuccessful.new(APP_STARTED_EVENT_RETRIES)
+
+        def initialize(
+          heartbeat_interval_seconds:,
+          metrics_aggregation_interval_seconds:,
+          emitter:,
+          metrics_manager:,
+          dependency_collection:,
+          enabled: true,
+          shutdown_timeout: Workers::Polling::DEFAULT_SHUTDOWN_TIMEOUT,
+          buffer_size: DEFAULT_BUFFER_MAX_SIZE
+        )
+          @emitter = emitter
+          @metrics_manager = metrics_manager
+          @dependency_collection = dependency_collection
+
+          @ticks_per_heartbeat = (heartbeat_interval_seconds / metrics_aggregation_interval_seconds).to_i
+          @current_ticks = 0
+
+          # Workers::Polling settings
+          self.enabled = enabled
+          # Workers::IntervalLoop settings
+          self.loop_base_interval = metrics_aggregation_interval_seconds
+          self.fork_policy = Core::Workers::Async::Thread::FORK_POLICY_STOP
+
+          @shutdown_timeout = shutdown_timeout
+          @buffer_size = buffer_size
+
+          self.buffer = buffer_klass.new(@buffer_size)
+        end
+
+        def start
+          return if !enabled? || forked?
+
+          # starts async worker
+          perform
+        end
+
+        def stop(force_stop = false, timeout = @shutdown_timeout)
+          buffer.close if running?
+
+          super
+        end
+
+        def enqueue(event)
+          return if !enabled? || forked?
+
+          buffer.push(event)
+        end
+
+        def sent_started_event?
+          TELEMETRY_STARTED_ONCE.success?
+        end
+
+        def failed_to_start?
+          TELEMETRY_STARTED_ONCE.failed?
+        end
+
+        private
+
+        def perform(*events)
+          return if !enabled? || forked?
+
+          started! unless sent_started_event?
+
+          metric_events = @metrics_manager.flush!
+          events = [] if events.nil?
+          flush_events(events + metric_events)
+
+          @current_ticks += 1
+          return if @current_ticks < @ticks_per_heartbeat
+
+          @current_ticks = 0
+          heartbeat!
+        end
+
+        def flush_events(events)
+          return if events.empty?
+          return if !enabled? || !sent_started_event?
+
+          Datadog.logger.debug { "Sending #{events&.count} telemetry events" }
+          send_event(Event::MessageBatch.new(events))
+        end
+
+        def heartbeat!
+          return if !enabled? || !sent_started_event?
+
+          send_event(Event::AppHeartbeat.new)
+        end
+
+        def started!
+          return unless enabled?
+
+          if failed_to_start?
+            Datadog.logger.debug('Telemetry app-started event exhausted retries, disabling telemetry worker')
+            disable!
+            return
+          end
+
+          TELEMETRY_STARTED_ONCE.run do
+            res = send_event(Event::AppStarted.new)
+
+            if res.ok?
+              Datadog.logger.debug('Telemetry app-started event is successfully sent')
+
+              send_event(Event::AppDependenciesLoaded.new) if @dependency_collection
+
+              true
+            else
+              Datadog.logger.debug('Error sending telemetry app-started event, retry after heartbeat interval...')
+              false
+            end
+          end
+        end
+
+        def send_event(event)
+          res = @emitter.request(event)
+
+          disable_on_not_found!(res)
+
+          res
+        end
+
+        def dequeue
+          buffer.pop
+        end
+
+        def work_pending?
+          run_loop? || !buffer.empty?
+        end
+
+        def buffer_klass
+          if Core::Environment::Ext::RUBY_ENGINE == 'ruby'
+            Core::Buffer::CRuby
+          else
+            Core::Buffer::ThreadSafe
+          end
+        end
+
+        def disable!
+          self.enabled = false
+          @metrics_manager.disable!
+        end
+
+        def disable_on_not_found!(response)
+          return unless response.not_found?
+
+          Datadog.logger.debug('Agent does not support telemetry; disabling future telemetry events.')
+          disable!
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/utils/only_once_successful.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require_relative 'only_once'
+
+module Datadog
+  module Core
+    module Utils
+      # Helper class to execute something with only one success.
+      #
+      # This is useful for cases where we want to ensure that a block of code is only executed once, and only if it
+      # succeeds. One such example is sending app-started telemetry event.
+      #
+      # Successful execution is determined by the return value of the block: any truthy value is considered success.
+      #
+      # Thread-safe when used correctly (e.g. be careful of races when lazily initializing instances of this class).
+      #
+      # Note: In its current state, this class is not Ractor-safe.
+      # In https://github.com/DataDog/dd-trace-rb/pull/1398#issuecomment-797378810 we have a discussion of alternatives,
+      # including an alternative implementation that is Ractor-safe once spent.
+      class OnlyOnceSuccessful < OnlyOnce
+        def initialize(limit = 0)
+          super()
+
+          @limit = limit
+          @failed = false
+          @retries = 0
+        end
+
+        def run
+          @mutex.synchronize do
+            return if @ran_once
+
+            result = yield
+            @ran_once = !!result
+
+            if !@ran_once && limited?
+              @retries += 1
+              check_limit!
+            end
+
+            result
+          end
+        end
+
+        def success?
+          @mutex.synchronize { @ran_once && !@failed }
+        end
+
+        def failed?
+          @mutex.synchronize { @ran_once && @failed }
+        end
+
+        private
+
+        def check_limit!
+          if @retries >= @limit
+            @failed = true
+            @ran_once = true
+          end
+        end
+
+        def limited?
+          !@limit.nil? && @limit.positive?
+        end
+
+        def reset_ran_once_state_for_tests
+          @mutex.synchronize do
+            @ran_once = false
+            @failed = false
+            @retries = 0
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative 'core/deprecations'
 require_relative 'core/extensions'
 
 # We must load core extensions to make certain global APIs
@@ -9,25 +10,7 @@ module Datadog
   # products. It is a dependency of each product. Contrast with Datadog::Kit
   # for higher-level features.
   module Core
-    class << self
-      # Records the occurrence of a deprecated operation in this library.
-      #
-      # Currently, these operations are logged to `Datadog.logger` at `warn` level.
-      #
-      # `disallowed_next_major` adds a message informing that the deprecated operation
-      # won't be allowed in the next major release.
-      #
-      # @yieldreturn [String] a String with the lazily evaluated deprecation message.
-      # @param [Boolean] disallowed_next_major whether this deprecation will be enforced in the next major release.
-      def log_deprecation(disallowed_next_major: true)
-        Datadog.logger.warn do
-          message = yield
-          message += ' This will be enforced in the next major release.' if disallowed_next_major
-          message
-        end
-        nil
-      end
-    end
+    extend Core::Deprecations
   end
 
   extend Core::Extensions

data/lib/datadog/opentelemetry/sdk/propagator.rb

@@ -41,8 +41,11 @@ module Datadog
           digest = @datadog_propagator.extract(carrier)
           return context unless digest
 
-          trace_id = to_otel_id(digest.trace_id)
-          span_id = to_otel_id(digest.span_id)
+          # Converts the {Numeric} Datadog id object to OpenTelemetry's byte array format.
+          # 128-bit unsigned, big-endian integer
+          trace_id = [digest.trace_id >> 64, digest.trace_id & 0xFFFFFFFFFFFFFFFF].pack('Q>Q>')
+          # 64-bit unsigned, big-endian integer
+          span_id = [digest.span_id].pack('Q>')
 
           if digest.trace_state || digest.trace_flags
             trace_flags = ::OpenTelemetry::Trace::TraceFlags.from_byte(digest.trace_flags)
@@ -78,14 +81,6 @@ module Datadog
         def fields
           []
         end
-
-        private
-
-        # Converts the {Numeric} Datadog id object to OpenTelemetry's byte array format.
-        # This method currently converts an unsigned 64-bit Integer to a binary String.
-        def to_otel_id(dd_id)
-          Array(dd_id).pack('Q')
-        end
       end
     end
   end

data/lib/datadog/opentelemetry/sdk/span_processor.rb

@@ -70,7 +70,8 @@ module Datadog
           if parent_context.trace
             Tracing.send(:tracer).send(:call_context).activate!(parent_context.ensure_trace)
           else
-            Tracing.continue_trace!(nil)
+            otel_trace_id = span.context.hex_trace_id.to_i(16)
+            Tracing.continue_trace!(Datadog::Tracing::TraceDigest.new(trace_id: otel_trace_id, span_remote: false))
           end
 
           datadog_span = start_datadog_span(span)
@@ -85,7 +86,6 @@ module Datadog
           name, kwargs = span_arguments(span, attributes)
 
           datadog_span = Tracing.trace(name, **kwargs)
-
           datadog_span.set_error([nil, span.status.description]) unless span.status.ok?
           datadog_span.set_tags(span.attributes)
 
@@ -143,6 +143,9 @@ module Datadog
 
           kwargs[:tags] = attributes.to_h
 
+          # DEV: The datadog span must have the same ID as the OpenTelemetry span
+          kwargs[:id] = span.context.hex_span_id.to_i(16)
+
           [name, kwargs]
         end
 

data/lib/datadog/profiling/collectors/code_provenance.rb

@@ -92,25 +92,38 @@ module Datadog
 
             seen_files << file_path
 
-            _, found_library = libraries_by_path.find { |library_path, _| file_path.start_with?(library_path) }
-            seen_libraries << found_library if found_library
+            # NOTE: Don't use .find, it allocates a lot more memory (see commit that added this note for details)
+            libraries_by_path.any? do |library_path, library|
+              seen_libraries << library if file_path.start_with?(library_path)
+            end
           end
         end
 
         # Represents metadata we have for a ruby gem
+        #
+        # Important note: This class gets encoded to JSON with the built-in JSON gem. But, we've found that in some
+        # buggy cases, some Ruby gems monkey patch the built-in JSON gem and forget to call #to_json, and instead
+        # encode this class instance-field-by-instance-field.
+        #
+        # Thus, this class was setup to match the JSON output. Take this into consideration if you are adding new
+        # fields. (Also, we have a spec for this)
         class Library
-          attr_reader :kind, :name, :version, :path
+          attr_reader :kind, :name, :version
 
           def initialize(kind:, name:, version:, path:)
             @kind = kind.freeze
             @name = name.dup.freeze
             @version = version.to_s.dup.freeze
-            @path = path.dup.freeze
+            @paths = [path.dup.freeze].freeze
             freeze
           end
 
           def to_json(arg = nil)
-            { kind: @kind, name: @name, version: @version, paths: [@path] }.to_json(arg)
+            { kind: @kind, name: @name, version: @version, paths: @paths }.to_json(arg)
+          end
+
+          def path
+            @paths.first
           end
         end
       end

data/lib/datadog/profiling/component.rb

@@ -75,6 +75,10 @@ module Datadog
         crashtracker = build_crashtracker(settings, transport)
         profiler = Profiling::Profiler.new(worker: worker, scheduler: scheduler, optional_crashtracker: crashtracker)
 
+        if dir_interruption_workaround_enabled?(settings, no_signals_workaround_enabled)
+          Datadog::Profiling::Ext::DirMonkeyPatches.apply!
+        end
+
         [profiler, { profiling_enabled: true }]
       end
 
@@ -397,8 +401,12 @@ module Datadog
 
       # See https://github.com/datadog/dd-trace-rb/issues/2976 for details.
       private_class_method def self.incompatible_passenger_version?
+        first_compatible_version = Gem::Version.new('6.0.19')
+
         if Gem.loaded_specs['passenger']
-          Gem.loaded_specs['passenger'].version < Gem::Version.new('6.0.19')
+          Gem.loaded_specs['passenger'].version < first_compatible_version
+        elsif defined?(PhusionPassenger::VERSION_STRING)
+          Gem::Version.new(PhusionPassenger::VERSION_STRING) < first_compatible_version
         else
           true
         end
@@ -445,6 +453,15 @@ module Datadog
           libmysqlclient_version < Gem::Version.new('5.0.0') &&
           header_version >= Gem::Version.new('10.0.0'))
       end
+
+      private_class_method def self.dir_interruption_workaround_enabled?(settings, no_signals_workaround_enabled)
+        return false if no_signals_workaround_enabled
+
+        # NOTE: In the future this method will evolve to check for Ruby versions affected and not apply the workaround
+        # when it's not needed but currently all known Ruby versions are affected.
+
+        settings.profiling.advanced.dir_interruption_workaround_enabled
+      end
     end
   end
 end