ddtrace 1.13.1 → 1.15.0

data/lib/datadog/core/transport/parcel.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Transport
+      # Data transfer object for generic data
+      # @abstract
+      module Parcel
+        attr_reader \
+          :data
+
+        def initialize(data)
+          @data = data
+        end
+
+        def encode_with(encoder)
+          raise NotImplementedError
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/transport/request.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Transport
+      # Defines request for transport operations
+      class Request
+        attr_reader \
+          :parcel
+
+        def initialize(parcel = nil)
+          @parcel = parcel
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/transport/response.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Transport
+      # Defines abstract response for transport operations
+      module Response
+        def payload
+          nil
+        end
+
+        def ok?
+          nil
+        end
+
+        def unsupported?
+          nil
+        end
+
+        def not_found?
+          nil
+        end
+
+        def client_error?
+          nil
+        end
+
+        def server_error?
+          nil
+        end
+
+        def internal_error?
+          nil
+        end
+
+        def inspect
+          "#{self.class} ok?:#{ok?} unsupported?:#{unsupported?}, " \
+            "not_found?:#{not_found?}, client_error?:#{client_error?}, " \
+            "server_error?:#{server_error?}, internal_error?:#{internal_error?}, " \
+            "payload:#{payload}"
+        end
+      end
+
+      # A generic error response for internal errors
+      class InternalErrorResponse
+        include Response
+
+        attr_reader :error
+
+        def initialize(error)
+          @error = error
+        end
+
+        def internal_error?
+          true
+        end
+
+        def inspect
+          "#{super}, error_type:#{error.class} error:#{error}"
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/workers/polling.rb

@@ -6,7 +6,7 @@ module Datadog
     module Workers
       # Adds polling (async looping) behavior to workers
       module Polling
-        SHUTDOWN_TIMEOUT = 1
+        DEFAULT_SHUTDOWN_TIMEOUT = 1
 
         def self.included(base)
           base.include(Workers::IntervalLoop)
@@ -21,7 +21,7 @@ module Datadog
           end
         end
 
-        def stop(force_stop = false, timeout = SHUTDOWN_TIMEOUT)
+        def stop(force_stop = false, timeout = DEFAULT_SHUTDOWN_TIMEOUT)
           if running?
             # Attempt graceful stop and wait
             stop_loop

data/lib/datadog/opentelemetry/api/context.rb

@@ -2,6 +2,7 @@
 
 require_relative 'trace/span'
 require_relative '../../tracing/trace_operation'
+require_relative '../trace'
 
 module Datadog
   module OpenTelemetry
@@ -66,9 +67,15 @@ module Datadog
         # @param [Object] value Object to be stored under key
         # @return [Context]
         def set_values(values)
-          trace = nil
-          if (current_span = values[CURRENT_SPAN_KEY])
-            trace = current_span.datadog_trace
+          if (current_span = values[CURRENT_SPAN_KEY]) && current_span.datadog_trace
+            # Because `#set_value` returns new context, we have to create
+            # a new copy of the active trace to ensure there's no conflict with
+            # other users of the active trace.
+            # It effectively becomes an internal trace propagation.
+            trace = Datadog::OpenTelemetry::Trace.start_trace_copy(
+              current_span.datadog_trace,
+              parent_span: current_span.datadog_span
+            )
           end
 
           existing_values = @trace && @trace.otel_values || {}

data/lib/datadog/opentelemetry/sdk/propagator.rb

@@ -81,8 +81,9 @@ module Datadog
         private
 
         # Converts the {Numeric} Datadog id object to OpenTelemetry's byte array format.
+        # This method currently converts an unsigned 64-bit Integer to a binary String.
         def to_otel_id(dd_id)
-          Array(dd_id).pack('S')
+          Array(dd_id).pack('Q')
         end
       end
     end

data/lib/datadog/opentelemetry/sdk/span_processor.rb

@@ -27,7 +27,7 @@ module Datadog
         #
         # @param [Span] span the {Span} that just ended.
         def on_finish(span)
-          span.datadog_span.finish
+          span.datadog_span.finish(ns_to_time(span.end_timestamp))
         end
 
         # Export all ended spans to the configured `Exporter` that have not yet
@@ -81,10 +81,22 @@ module Datadog
           kind = span.kind || 'internal'
           tags[Tracing::Metadata::Ext::TAG_KIND] = kind
 
-          datadog_span = Tracing.trace(span.name, tags: tags)
+          datadog_span = Tracing.trace(
+            span.name,
+            tags: tags,
+            start_time: ns_to_time(span.start_timestamp)
+          )
+
           datadog_span.set_error([nil, span.status.description]) unless span.status.ok?
+          datadog_span.set_tags(span.attributes)
+
           datadog_span
         end
+
+        # From nanoseconds, used by OpenTelemetry, to a {Time} object, used by the Datadog Tracer.
+        def ns_to_time(timestamp_ns)
+          Time.at(timestamp_ns / 1000000000.0)
+        end
       end
     end
   end

data/lib/datadog/opentelemetry/sdk/trace/span.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Datadog
+  module OpenTelemetry
+    module Trace
+      # Stores associated Datadog entities to the OpenTelemetry Span.
+      module Span
+        # Attributes are equivalent to span tags and metrics.
+        def set_attribute(key, value)
+          res = super
+          # Attributes can get dropped or their values truncated by `super`
+          datadog_set_attribute(key)
+          res
+        end
+
+        # `alias` performed to match {OpenTelemetry::SDK::Trace::Span} aliasing upstream
+        alias []= set_attribute
+
+        # Attributes are equivalent to span tags and metrics.
+        def add_attributes(attributes)
+          res = super
+          # Attributes can get dropped or their values truncated by `super`
+          attributes.each { |key, _| datadog_set_attribute(key) }
+          res
+        end
+
+        # Captures changes to span error state.
+        def status=(s)
+          super
+
+          return unless status # Return if status are currently disabled by OpenTelemetry.
+          return unless (span = datadog_span)
+
+          # Status code can only change into an error state.
+          # Other change operations should be ignored.
+          span.set_error(status.description) if status && status.code == ::OpenTelemetry::Trace::Status::ERROR
+        end
+
+        private
+
+        def datadog_set_attribute(key)
+          # Return if attributes are currently disabled by OpenTelemetry.
+          return unless defined?(@attributes) && @attributes
+          return unless (span = datadog_span)
+
+          # DEV: Accesses `@attributes` directly, since using `#attributes`
+          # DEV: clones the hash, causing unnecessary overhead.
+          if @attributes.key?(key)
+            value = @attributes[key]
+            span.set_tag(key, value)
+
+            span.service = value if key == 'service.name'
+          else
+            span.clear_tag(key)
+
+            if key == 'service.name'
+              # By removing the service name, we set it to the fallback default,
+              # effectively removing the `service` attribute from OpenTelemetry's perspective.
+              span.service = Datadog.send(:components).tracer.default_service
+            end
+          end
+        end
+
+        ::OpenTelemetry::SDK::Trace::Span.prepend(self)
+      end
+    end
+  end
+end

data/lib/datadog/opentelemetry/trace.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require_relative '../tracing/trace_digest'
+
+module Datadog
+  module OpenTelemetry
+    # OpenTelemetry utilities related to the respective Datadog trace.
+    module Trace
+      class << self
+        # Creates a new TraceOperation object that can be attached to a new
+        # OpenTelemetry span.
+        # If `parent_span` is provided, then that span is set as the currently
+        # active parent span.
+        #
+        # @return [TraceOperation]
+        def start_trace_copy(trace, parent_span: nil)
+          digest = if parent_span
+                     digest_with_parent_span(trace, parent_span)
+                   else
+                     trace.to_digest
+                   end
+
+          # Create a new TraceOperation, attached to the current Datadog Tracer.
+          Datadog::Tracing.continue_trace!(digest)
+        end
+
+        private
+
+        # Creates a TraceDigest with the active span modified.
+        # This supports the implementation of `OpenTelemetry::Trace.context_with_span`,
+        # which allows you to specific any span as the arbitrary parent of a new span.
+        def digest_with_parent_span(trace, parent_span)
+          digest = trace.to_digest
+
+          Tracing::TraceDigest.new(
+            span_id: parent_span.id,
+            span_name: parent_span.name,
+            span_resource: parent_span.resource,
+            span_service: parent_span.service,
+            span_type: parent_span.type,
+            trace_distributed_tags: digest.trace_distributed_tags,
+            trace_hostname: digest.trace_hostname,
+            trace_id: digest.trace_id,
+            trace_name: digest.trace_name,
+            trace_origin: digest.trace_origin,
+            trace_process_id: digest.trace_process_id,
+            trace_resource: digest.trace_resource,
+            trace_runtime_id: digest.trace_runtime_id,
+            trace_sampling_priority: digest.trace_sampling_priority,
+            trace_service: digest.trace_service,
+            trace_state: digest.trace_state,
+            trace_state_unknown_fields: digest.trace_state_unknown_fields,
+          ).freeze
+        end
+      end
+    end
+  end
+end

data/lib/datadog/opentelemetry.rb

@@ -16,6 +16,7 @@ require_relative 'opentelemetry/api/context'
 # DEV: This is different from our existing OpenTracer API, but there are many safety
 # DEV: features built into Contrib instrumentation today.
 require_relative 'opentelemetry/sdk/configurator' if defined?(OpenTelemetry::SDK)
+require_relative 'opentelemetry/sdk/trace/span' if defined?(OpenTelemetry::SDK)
 
 module Datadog
   # Datadog OpenTelemetry integration.

data/lib/datadog/opentracer.rb

@@ -22,3 +22,12 @@ require_relative 'opentracer/global_tracer'
 
 # Modify the OpenTracing module functions
 ::OpenTracing.singleton_class.prepend(Datadog::OpenTracer::GlobalTracer)
+
+module Datadog
+  # Datadog OpenTracing integration.
+  # DEV: This module should be named `Datadog::OpenTracing` to match the gem (`opentracing`).
+  module OpenTracer
+    # Used by Telemetry to decide if OpenTracing instrumentation is enabled
+    LOADED = true
+  end
+end

data/lib/datadog/profiling/collectors/cpu_and_wall_time_worker.rb

@@ -14,25 +14,15 @@ module Datadog
         public
 
         def initialize(
-          recorder:,
-          max_frames:,
-          tracer:,
-          endpoint_collection_enabled:,
           gc_profiling_enabled:,
           allocation_counting_enabled:,
           no_signals_workaround_enabled:,
-          timeline_enabled:,
-          thread_context_collector: ThreadContext.new(
-            recorder: recorder,
-            max_frames: max_frames,
-            tracer: tracer,
-            endpoint_collection_enabled: endpoint_collection_enabled,
-            timeline_enabled: timeline_enabled,
-          ),
+          thread_context_collector:,
           idle_sampling_helper: IdleSamplingHelper.new,
           # **NOTE**: This should only be used for testing; disabling the dynamic sampling rate will increase the
           # profiler overhead!
-          dynamic_sampling_rate_enabled: true
+          dynamic_sampling_rate_enabled: true,
+          allocation_sample_every: 0 # Currently only for testing; Setting this to > 0 can add a lot of overhead!
         )
           unless dynamic_sampling_rate_enabled
             Datadog.logger.warn(
@@ -40,6 +30,13 @@ module Datadog
             )
           end
 
+          if allocation_counting_enabled && allocation_sample_every > 0
+            Datadog.logger.warn(
+              "Enabled experimental allocation profiling: allocation_sample_every=#{allocation_sample_every}. This is " \
+              'experimental, not recommended, and will increase overhead!'
+            )
+          end
+
           self.class._native_initialize(
             self,
             thread_context_collector,
@@ -48,6 +45,7 @@ module Datadog
             allocation_counting_enabled,
             no_signals_workaround_enabled,
             dynamic_sampling_rate_enabled,
+            allocation_sample_every,
           )
           @worker_thread = nil
           @failure_exception = nil
@@ -84,11 +82,7 @@ module Datadog
           true
         end
 
-        # TODO: Provided only for compatibility with the API for Collectors::OldStack used in the Profiler class.
-        # Can be removed once we remove OldStack.
-        def enabled=(_); end
-
-        def stop(*_)
+        def stop
           @start_stop_mutex.synchronize do
             Datadog.logger.debug('Requesting CpuAndWallTimeWorker thread shut down')
 

data/lib/datadog/profiling/collectors/idle_sampling_helper.rb

@@ -48,7 +48,7 @@ module Datadog
           true
         end
 
-        def stop(*_unused)
+        def stop
           @start_stop_mutex.synchronize do
             Datadog.logger.debug('Requesting IdleSamplingHelper thread shut down')
 

data/lib/datadog/profiling/collectors/thread_context.rb

@@ -14,7 +14,14 @@ module Datadog
       #
       # Methods prefixed with _native_ are implemented in `collectors_thread_context.c`
       class ThreadContext
-        def initialize(recorder:, max_frames:, tracer:, endpoint_collection_enabled:, timeline_enabled:)
+        def initialize(
+          recorder:,
+          max_frames:,
+          tracer:,
+          endpoint_collection_enabled:,
+          timeline_enabled:,
+          allocation_type_enabled: true
+        )
           tracer_context_key = safely_extract_context_key_from(tracer)
           self.class._native_initialize(
             self,
@@ -23,6 +30,7 @@ module Datadog
             tracer_context_key,
             endpoint_collection_enabled,
             timeline_enabled,
+            allocation_type_enabled,
           )
         end
 

data/lib/datadog/profiling/component.rb

@@ -7,7 +7,11 @@ module Datadog
       # Passing in a `nil` tracer is supported and will disable the following profiling features:
       # * Code Hotspots panel in the trace viewer, as well as scoping a profile down to a span
       # * Endpoint aggregation in the profiler UX, including normalization (resource per endpoint call)
-      def self.build_profiler_component(settings:, agent_settings:, optional_tracer:) # rubocop:disable Metrics/MethodLength
+      def self.build_profiler_component(settings:, agent_settings:, optional_tracer:)
+        require_relative '../profiling/diagnostics/environment_logger'
+
+        Profiling::Diagnostics::EnvironmentLogger.collect_and_log!
+
         return unless settings.profiling.enabled
 
         # Workaround for weird dependency direction: the Core::Configuration::Components class currently has a
@@ -30,98 +34,53 @@ module Datadog
         require_relative '../profiling'
         return unless Profiling.supported?
 
-        unless defined?(Profiling::Tasks::Setup)
-          # In #1545 a user reported a NameError due to this constant being uninitialized
-          # I've documented my suspicion on why that happened in
-          # https://github.com/DataDog/dd-trace-rb/issues/1545#issuecomment-856049025
-          #
-          # > Thanks for the info! It seems to feed into my theory: there's two moments in the code where we check if
-          # > profiler is "supported": 1) when loading ddtrace (inside preload) and 2) when starting the profile
-          # > after Datadog.configure gets run.
-          # > The problem is that the code assumes that both checks 1) and 2) will always reach the same conclusion:
-          # > either profiler is supported, or profiler is not supported.
-          # > In the problematic case, it looks like in your case check 1 decides that profiler is not
-          # > supported => doesn't load it, and then check 2 decides that it is => assumes it is loaded and tries to
-          # > start it.
-          #
-          # I was never able to validate if this was the issue or why exactly .supported? would change its mind BUT
-          # just in case it happens again, I've left this check which avoids breaking the user's application AND
-          # would instead direct them to report it to us instead, so that we can investigate what's wrong.
-          #
-          # TODO: As of June 2021, most checks in .supported? are related to the google-protobuf gem; so it's
-          # very likely that it was the origin of the issue we saw. Thus, if, as planned we end up moving away from
-          # protobuf OR enough time has passed and no users saw the issue again, we can remove this check altogether.
-          Datadog.logger.error(
-            'Profiling was marked as supported and enabled, but setup task was not loaded properly. ' \
-            'Please report this at https://github.com/DataDog/dd-trace-rb/blob/master/CONTRIBUTING.md#found-a-bug'
-          )
-
-          return
-        end
-
-        # Load extensions needed to support some of the Profiling features
+        # Activate forking extensions
         Profiling::Tasks::Setup.new.run
 
         # NOTE: Please update the Initialization section of ProfilingDevelopment.md with any changes to this method
 
-        no_signals_workaround_enabled = false
+        no_signals_workaround_enabled = no_signals_workaround_enabled?(settings)
+        timeline_enabled = settings.profiling.advanced.experimental_timeline_enabled
 
-        if enable_new_profiler?(settings)
-          no_signals_workaround_enabled = no_signals_workaround_enabled?(settings)
-
-          recorder = Datadog::Profiling::StackRecorder.new(
-            cpu_time_enabled: RUBY_PLATFORM.include?('linux'), # Only supported on Linux currently
-            alloc_samples_enabled: false, # Always disabled for now -- work in progress
-          )
-          collector = Datadog::Profiling::Collectors::CpuAndWallTimeWorker.new(
-            recorder: recorder,
-            max_frames: settings.profiling.advanced.max_frames,
-            tracer: optional_tracer,
-            endpoint_collection_enabled: settings.profiling.advanced.endpoint.collection.enabled,
-            gc_profiling_enabled: enable_gc_profiling?(settings),
-            allocation_counting_enabled: settings.profiling.advanced.allocation_counting_enabled,
-            no_signals_workaround_enabled: no_signals_workaround_enabled,
-            timeline_enabled: settings.profiling.advanced.experimental_timeline_enabled,
-          )
-        else
-          load_pprof_support
+        recorder = Datadog::Profiling::StackRecorder.new(
+          cpu_time_enabled: RUBY_PLATFORM.include?('linux'), # Only supported on Linux currently
+          alloc_samples_enabled: false, # Always disabled for now -- work in progress
+        )
+        thread_context_collector = Datadog::Profiling::Collectors::ThreadContext.new(
+          recorder: recorder,
+          max_frames: settings.profiling.advanced.max_frames,
+          tracer: optional_tracer,
+          endpoint_collection_enabled: settings.profiling.advanced.endpoint.collection.enabled,
+          timeline_enabled: timeline_enabled,
+        )
+        worker = Datadog::Profiling::Collectors::CpuAndWallTimeWorker.new(
+          gc_profiling_enabled: enable_gc_profiling?(settings),
+          allocation_counting_enabled: settings.profiling.advanced.allocation_counting_enabled,
+          no_signals_workaround_enabled: no_signals_workaround_enabled,
+          thread_context_collector: thread_context_collector,
+          allocation_sample_every: 0,
+        )
 
-          recorder = build_profiler_old_recorder(settings)
-          collector = build_profiler_oldstack_collector(settings, recorder, optional_tracer)
-        end
+        internal_metadata = {
+          no_signals_workaround_enabled: no_signals_workaround_enabled,
+          timeline_enabled: timeline_enabled,
+        }.freeze
 
-        exporter = build_profiler_exporter(settings, recorder, no_signals_workaround_enabled: no_signals_workaround_enabled)
+        exporter = build_profiler_exporter(settings, recorder, internal_metadata: internal_metadata)
         transport = build_profiler_transport(settings, agent_settings)
         scheduler = Profiling::Scheduler.new(exporter: exporter, transport: transport)
 
-        Profiling::Profiler.new([collector], scheduler)
+        Profiling::Profiler.new(worker: worker, scheduler: scheduler)
       end
 
-      private_class_method def self.build_profiler_old_recorder(settings)
-        Profiling::OldRecorder.new([Profiling::Events::StackSample], settings.profiling.advanced.max_events)
-      end
-
-      private_class_method def self.build_profiler_exporter(settings, recorder, no_signals_workaround_enabled:)
+      private_class_method def self.build_profiler_exporter(settings, recorder, internal_metadata:)
         code_provenance_collector =
           (Profiling::Collectors::CodeProvenance.new if settings.profiling.advanced.code_provenance_enabled)
 
         Profiling::Exporter.new(
           pprof_recorder: recorder,
           code_provenance_collector: code_provenance_collector,
-          no_signals_workaround_enabled: no_signals_workaround_enabled,
-        )
-      end
-
-      private_class_method def self.build_profiler_oldstack_collector(settings, old_recorder, tracer)
-        trace_identifiers_helper = Profiling::TraceIdentifiers::Helper.new(
-          tracer: tracer,
-          endpoint_collection_enabled: settings.profiling.advanced.endpoint.collection.enabled
-        )
-
-        Profiling::Collectors::OldStack.new(
-          old_recorder,
-          trace_identifiers_helper: trace_identifiers_helper,
-          max_frames: settings.profiling.advanced.max_frames
+          internal_metadata: internal_metadata,
         )
       end
 
@@ -151,17 +110,6 @@ module Datadog
         end
       end
 
-      private_class_method def self.enable_new_profiler?(settings)
-        if settings.profiling.advanced.force_enable_legacy_profiler
-          Datadog.logger.warn(
-            'Legacy profiler has been force-enabled via configuration. Do not use unless instructed to by support.'
-          )
-          return false
-        end
-
-        true
-      end
-
       private_class_method def self.no_signals_workaround_enabled?(settings) # rubocop:disable Metrics/MethodLength
         setting_value = settings.profiling.advanced.no_signals_workaround_enabled
         legacy_ruby_that_should_use_workaround = RUBY_VERSION.start_with?('2.3.', '2.4.', '2.5.')
@@ -289,19 +237,6 @@ module Datadog
           true
         end
       end
-
-      # The old profiler's pprof support conflicts with the ruby-cloud-profiler gem.
-      #
-      # This is not a problem for almost all customers, since we now default everyone to use the new CPU Profiling 2.0
-      # profiler. But the issue was still triggered, because currently we still _load_ both the old and new profiling
-      # code paths.
-      #
-      # To work around this issue, and because we plan on deleting the old profiler soon, rather than poking at the
-      # pprof support code, we only load the conflicting file when the old profiler is in use. This way customers using
-      # the new profiler will not be affected by the issue any longer.
-      private_class_method def self.load_pprof_support
-        require_relative 'pprof/pprof_pb'
-      end
     end
   end
 end

data/lib/datadog/profiling/diagnostics/environment_logger.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require 'date'
+require 'json'
+require 'rbconfig'
+require_relative '../../core/diagnostics/environment_logger'
+
+module Datadog
+  module Profiling
+    module Diagnostics
+      # Collects and logs Profiling diagnostic information
+      module EnvironmentLogger
+        extend Core::Diagnostics::EnvironmentLogging
+
+        def self.collect_and_log!
+          log_once! do
+            data = EnvironmentCollector.collect_config!
+            log_configuration!('PROFILING', data.to_json)
+          end
+        rescue => e
+          logger.warn("Failed to collect profiling environment information: #{e} Location: #{Array(e.backtrace).first}")
+        end
+      end
+
+      # Collects environment information for Profiling diagnostic logging
+      module EnvironmentCollector
+        def self.collect_config!(*args)
+          {
+            profiling_enabled: profiling_enabled
+          }
+        end
+
+        def self.profiling_enabled
+          !!Datadog.configuration.profiling.enabled
+        end
+      end
+    end
+  end
+end