datadog 2.4.0 → 2.44.0

data/lib/datadog/tracing/trace_segment.rb

@@ -1,21 +1,21 @@
 # frozen_string_literal: true
 
-require_relative '../core/runtime/ext'
-require_relative '../core/utils/safe_dup'
+require_relative "../core/runtime/ext"
+require_relative "../core/utils/safe_dup"
 
-require_relative 'sampling/ext'
-require_relative 'metadata/ext'
-require_relative 'metadata/tagging'
-require_relative 'utils'
+require_relative "sampling/ext"
+require_relative "metadata/ext"
+require_relative "metadata/tagging"
+require_relative "utils"
 
 module Datadog
   module Tracing
     # Serializable construct representing a trace
     # @public_api
     class TraceSegment
-      TAG_NAME = 'name'
-      TAG_RESOURCE = 'resource'
-      TAG_SERVICE = 'service'
+      TAG_NAME = "name"
+      TAG_RESOURCE = "resource"
+      TAG_SERVICE = "service"
 
       attr_reader \
         :id,
@@ -34,7 +34,8 @@ module Datadog
         :sampling_decision_maker,
         :sampling_priority,
         :service,
-        :profiling_enabled
+        :profiling_enabled,
+        :apm_tracing_enabled
 
       # rubocop:disable Metrics/CyclomaticComplexity
       # rubocop:disable Metrics/PerceivedComplexity
@@ -58,7 +59,8 @@ module Datadog
         service: nil,
         tags: nil,
         metrics: nil,
-        profiling_enabled: nil
+        profiling_enabled: nil,
+        apm_tracing_enabled: nil
       )
         @id = id
         @root_span_id = root_span_id
@@ -66,8 +68,8 @@ module Datadog
 
         # Does not make an effort to move metrics out of tags
         # The caller is expected to have done that
-        @meta = (tags && tags.dup) || {}
-        @metrics = (metrics && metrics.dup) || {}
+        @meta = tags&.dup || {}
+        @metrics = metrics&.dup || {}
 
         # Set well-known tags, defaulting to getting the values from tags
         @agent_sample_rate = agent_sample_rate || agent_sample_rate_tag
@@ -85,6 +87,7 @@ module Datadog
         @sampling_priority = sampling_priority || sampling_priority_tag
         @service = Core::Utils::SafeDup.frozen_or_dup(service || service_tag)
         @profiling_enabled = profiling_enabled
+        @apm_tracing_enabled = apm_tracing_enabled
       end
       # rubocop:enable Metrics/PerceivedComplexity
       # rubocop:enable Metrics/CyclomaticComplexity
@@ -128,8 +131,7 @@ module Datadog
       end
 
       def sampled?
-        sampling_priority == Sampling::Ext::Priority::AUTO_KEEP \
-          || sampling_priority == Sampling::Ext::Priority::USER_KEEP
+        [Sampling::Ext::Priority::AUTO_KEEP, Sampling::Ext::Priority::USER_KEEP].include?(sampling_priority)
       end
 
       # Returns the high order part of the trace id as a hexadecimal string; the most significant 64 bits.
@@ -138,7 +140,7 @@ module Datadog
       def high_order_tid
         high_order = Tracing::Utils::TraceId.to_high_order(@id)
 
-        format('%016x', high_order) if high_order != 0
+        format("%016x", high_order) if high_order != 0
       end
 
       protected

data/lib/datadog/tracing/tracer.rb

@@ -1,20 +1,20 @@
 # frozen_string_literal: true
 
-require_relative '../core/environment/ext'
-require_relative '../core/environment/socket'
-
-require_relative 'correlation'
-require_relative 'event'
-require_relative 'flush'
-require_relative 'context_provider'
-require_relative 'sampling/all_sampler'
-require_relative 'sampling/rule_sampler'
-require_relative 'sampling/priority_sampler'
-require_relative 'sampling/span/sampler'
-require_relative 'span_operation'
-require_relative 'trace_digest'
-require_relative 'trace_operation'
-require_relative 'writer'
+require_relative "../core/environment/ext"
+require_relative "../core/environment/socket"
+
+require_relative "correlation"
+require_relative "event"
+require_relative "flush"
+require_relative "context_provider"
+require_relative "sampling/all_sampler"
+require_relative "sampling/rule_sampler"
+require_relative "sampling/priority_sampler"
+require_relative "sampling/span/sampler"
+require_relative "span_operation"
+require_relative "trace_digest"
+require_relative "trace_operation"
+require_relative "writer"
 
 module Datadog
   module Tracing
@@ -28,7 +28,8 @@ module Datadog
         :provider,
         :sampler,
         :span_sampler,
-        :tags
+        :tags,
+        :logger
 
       attr_accessor \
         :default_service,
@@ -48,26 +49,39 @@ module Datadog
       # @param tags [Hash] default tags added to all spans
       # @param writer [Datadog::Tracing::Writer] consumes traces returned by the provided +trace_flush+
       def initialize(
+        # rubocop:disable Style/KeywordParametersOrder
+        # https://github.com/rubocop/rubocop/issues/13933
         trace_flush: Flush::Finished.new,
-        context_provider: DefaultContextProvider.new,
+        context_provider: nil,
         default_service: Core::Environment::Ext::FALLBACK_SERVICE_NAME,
         enabled: true,
+        logger: Datadog.logger,
+        otel_thread_context: nil,
         sampler: Sampling::PrioritySampler.new(
           base_sampler: Sampling::AllSampler.new,
           post_sampler: Sampling::RuleSampler.new
         ),
         span_sampler: Sampling::Span::Sampler.new,
         tags: {},
-        writer: Writer.new
+        # writer is not defaulted because creating it requires agent_settings,
+        # which we do not have here and otherwise do not need.
+        writer:
+        # rubocop:enable Style/KeywordParametersOrder
       )
         @trace_flush = trace_flush
         @default_service = default_service
         @enabled = enabled
-        @provider = context_provider
+        @logger = logger
         @sampler = sampler
         @span_sampler = span_sampler
         @tags = tags
         @writer = writer
+
+        @otel_thread_context = otel_thread_context
+
+        @provider = context_provider || DefaultContextProvider.new(
+          otel_thread_context: otel_thread_context
+        )
       end
 
       # Return a {Datadog::Tracing::SpanOperation span_op} and {Datadog::Tracing::TraceOperation trace_op}
@@ -121,7 +135,6 @@ module Datadog
       # @yield Optional block where new newly created {Datadog::Tracing::SpanOperation} captures the execution.
       # @yieldparam [Datadog::Tracing::SpanOperation] span_op the newly created and active [Datadog::Tracing::SpanOperation]
       # @yieldparam [Datadog::Tracing::TraceOperation] trace_op the active [Datadog::Tracing::TraceOperation]
-      # rubocop:disable Metrics/MethodLength
       def trace(
         name,
         continue_from: nil,
@@ -141,12 +154,12 @@ module Datadog
           context = call_context
           active_trace = context.active_trace
           trace = if continue_from || active_trace.nil?
-                    start_trace(continue_from: continue_from)
-                  else
-                    active_trace
-                  end
-        rescue StandardError => e
-          Datadog.logger.debug { "Failed to trace: #{e}" }
+            start_trace(continue_from: continue_from)
+          else
+            active_trace
+          end
+        rescue => e
+          logger.debug { "Failed to trace: #{e.class}: #{e.message}" }
 
           # Tracing failed: fallback and run code without tracing.
           return skip_trace(name, &block)
@@ -186,7 +199,6 @@ module Datadog
           )
         end
       end
-      # rubocop:enable Metrics/MethodLength
 
       # Set the given key / value tag pair at the tracer level. These tags will be
       # appended to each span created by the tracer. Keys and values must be strings.
@@ -217,7 +229,7 @@ module Datadog
       # @return [nil] if no trace is active, and thus no span is active
       def active_span(key = nil)
         trace = active_trace(key)
-        trace.active_span if trace
+        trace&.active_span
       end
 
       # Information about the currently active trace.
@@ -234,10 +246,21 @@ module Datadog
         trace.to_correlation
       end
 
-      # Setup a new trace to continue from where another
+      # Setup a new trace execution context to continue from where another
       # trace left off.
+      # This is useful to continue distributed or async traces.
       #
-      # Used to continue distributed or async traces.
+      # The first span created in the restored context is a direct child of the
+      # active span from when the {Datadog::Tracing::TraceDigest} was created.
+      #
+      # When no block is given, the trace context is restored in the current thread.
+      # It remains active until the first span created in this restored context is finished.
+      # After that, if a new span is created, it start a new, unrelated trace.
+      #
+      # When a block is given, the trace context is restored inside the block execution.
+      # It remains active until the block ends, even when the first span created inside
+      # the block finishes. This means that multiple spans can be direct children of the
+      # active span from when the {Datadog::Tracing::TraceDigest} was created.
       #
       # @param [Datadog::Tracing::TraceDigest] digest continue from the {Datadog::Tracing::TraceDigest}.
       # @param [Thread] key Thread to retrieve trace from. Defaults to current thread. For internal use only.
@@ -253,22 +276,52 @@ module Datadog
         # Start a new trace from the digest
         context = call_context(key)
         original_trace = active_trace(key)
-        trace = start_trace(continue_from: digest)
+        # When we want the trace to be bound to a block, we cannot let
+        # it auto finish when the local root span finishes. This would
+        # create mutiple traces inside the block. Instead, we'll
+        # expliclity finish the trace after the block finishes.
+        auto_finish = !block
+
+        trace = start_trace(continue_from: digest, auto_finish: auto_finish)
 
         # If block hasn't been given; we need to manually deactivate
         # this trace. Subscribe to the trace finished event to do this.
         subscribe_trace_deactivation!(context, trace, original_trace) unless block
 
-        context.activate!(trace, &block)
+        if block
+          # When a block is given, the trace will be active until the block finishes.
+          context.activate!(trace) do
+            yield
+          ensure # We have to flush even when an error occurs
+            # On block completion, force the trace to finish and flush its finished spans.
+            # Unfinished spans are lost as the {TraceOperation} has ended.
+            trace.finish!
+            flush_trace(trace)
+          end
+        else
+          # Otherwise, the trace will be bound to the current thread after this point
+          context.activate!(trace)
+        end
       end
 
       # Sample a span, tagging the trace as appropriate.
       def sample_trace(trace_op)
-        begin
-          @sampler.sample!(trace_op)
-        rescue StandardError => e
-          SAMPLE_TRACE_LOG_ONLY_ONCE.run do
-            Datadog.logger.warn { "Failed to sample trace: #{e.class.name} #{e} at #{Array(e.backtrace).first}" }
+        @sampler.sample!(trace_op) if trace_op.sampling_priority.nil?
+      rescue => e
+        SAMPLE_TRACE_LOG_ONLY_ONCE.run do
+          logger.warn { "Failed to sample trace: #{e.class}: #{e.message} at #{Array(e.backtrace).first}" }
+        end
+      end
+
+      def reconsider_trace_sampling_on_resource(trace_op)
+        return unless trace_op.reconsider_resource_sample?
+        return unless @sampler.respond_to?(:reconsider_sample_resource!)
+
+        @sampler.reconsider_sample_resource!(trace_op)
+      rescue => e
+        RECONSIDER_RESOURCE_SAMPLE_TRACE_LOG_ONLY_ONCE.run do
+          logger.warn do
+            "Failed to reconsider trace sampling: #{e.class}: #{e.message} at #{Array(e.backtrace).first}"
           end
         end
       end
@@ -289,11 +342,15 @@ module Datadog
         #       description of and constraints on arguments.
         # rubocop:disable Lint/UselessMethodDefinition
         def publish(trace)
-          super(trace)
+          super
         end
         # rubocop:enable Lint/UselessMethodDefinition
       end
 
+      def after_fork
+        @otel_thread_context&.after_fork
+      end
+
       # Shorthand that calls the `shutdown!` method of a registered worker.
       # It's useful to ensure that the Trace Buffer is properly flushed before
       # shutting down the application.
@@ -305,9 +362,10 @@ module Datadog
       #
       #   tracer.shutdown!
       def shutdown!
+        @otel_thread_context&.clear
         return unless @enabled
 
-        @writer.stop if @writer
+        @writer&.stop
       end
 
       private
@@ -324,32 +382,41 @@ module Datadog
         @provider.context(key)
       end
 
-      def build_trace(digest = nil)
-        # Resolve hostname if configured
-        hostname = Core::Environment::Socket.hostname if Datadog.configuration.tracing.report_hostname
-        hostname = hostname && !hostname.empty? ? hostname : nil
+      def build_trace(digest, auto_finish)
+        hostname = Core::Environment::Socket.resolved_hostname(Datadog.configuration)
 
         if digest
+          propagate_sampling = propagate_sampling_priority?(upstream_tags: digest.trace_distributed_tags)
+          sampling_priority = digest.trace_sampling_priority if propagate_sampling
+          trace_state = Distributed::TraceState.from_digest(digest, propagate_sampling: propagate_sampling)
           TraceOperation.new(
+            logger: logger,
             hostname: hostname,
             profiling_enabled: profiling_enabled,
+            apm_tracing_enabled: apm_tracing_enabled,
             id: digest.trace_id,
             origin: digest.trace_origin,
             parent_span_id: digest.span_id,
-            sampling_priority: digest.trace_sampling_priority,
+            sampling_priority: sampling_priority,
+            span_links: digest.span_links,
             # Distributed tags are just regular trace tags with special meaning to Datadog
             tags: digest.trace_distributed_tags,
-            trace_state: digest.trace_state,
-            trace_state_unknown_fields: digest.trace_state_unknown_fields,
+            trace_state: trace_state,
             remote_parent: digest.span_remote,
-            tracer: self
+            distributed_sampling_priority: !!sampling_priority,
+            tracer: self,
+            baggage: digest.baggage,
+            auto_finish: auto_finish
           )
         else
           TraceOperation.new(
+            logger: logger,
             hostname: hostname,
             profiling_enabled: profiling_enabled,
+            apm_tracing_enabled: apm_tracing_enabled,
             remote_parent: false,
-            tracer: self
+            tracer: self,
+            auto_finish: auto_finish
           )
         end
       end
@@ -362,17 +429,42 @@ module Datadog
           event_span_op.service ||= @default_service
         end
 
+        events.span_before_finish.subscribe do |event_span_op, _event_trace_op|
+          if event_span_op.service && event_span_op.service != @default_service
+            event_span_op.set_tag(Tracing::Metadata::Ext::TAG_BASE_SERVICE, @default_service)
+            event_span_op.set_tag(Tracing::Metadata::Ext::TAG_SVC_SRC, Tracing::Metadata::Ext::SVC_SRC_MANUAL) unless event_span_op.get_tag(Tracing::Metadata::Ext::TAG_SVC_SRC)
+          else
+            event_span_op.send(:meta).delete(Tracing::Metadata::Ext::TAG_SVC_SRC)
+          end
+        end
+
+        events.trace_propagated.subscribe do |event_trace_op|
+          sample_trace(event_trace_op)
+        end
+
         events.span_finished.subscribe do |event_span, event_trace_op|
+          sample_trace(trace_op) if event_trace_op.sampling_priority.nil?
           sample_span(event_trace_op, event_span)
           flush_trace(event_trace_op)
         end
+
+        # Conditionally subscribe to the less common sampling rules below, to avoid
+        # measurable unnecessary performance overhead when they are not present.
+
+        if @sampler.respond_to?(:resource_sampling?) && @sampler.resource_sampling?
+          events.trace_resource_change.subscribe do |event_trace_op|
+            reconsider_trace_sampling_on_resource(event_trace_op)
+          end
+        end
+
+        @otel_thread_context&.subscribe_to_tracer_events!(events)
       end
 
       # Creates a new TraceOperation, with events bounds to this Tracer instance.
       # @return [TraceOperation]
-      def start_trace(continue_from: nil)
+      def start_trace(continue_from: nil, auto_finish: true)
         # Build a new trace using digest if provided.
-        trace = build_trace(continue_from)
+        trace = build_trace(continue_from, auto_finish)
 
         # Bind trace events: sample trace, set default service, flush spans.
         bind_trace_events!(trace)
@@ -396,17 +488,18 @@ module Datadog
       )
         trace = _trace || start_trace(continue_from: continue_from)
 
-        events = SpanOperation::Events.new
+        events = SpanOperation::Events.new(logger: logger)
 
         if block
           # Ignore start time if a block has been given
           trace.measure(
             name,
+            logger: logger,
             events: events,
             on_error: on_error,
             resource: resource,
             service: service,
-            tags: resolve_tags(tags),
+            tags: resolve_tags(tags, service),
             type: type,
             id: id,
             &block
@@ -415,12 +508,13 @@ module Datadog
           # Return the new span
           span = trace.build_span(
             name,
+            logger: logger,
             events: events,
             on_error: on_error,
             resource: resource,
             service: service,
             start_time: start_time,
-            tags: resolve_tags(tags),
+            tags: resolve_tags(tags, service),
             type: type,
             id: id
           )
@@ -430,10 +524,10 @@ module Datadog
           span
         end
       end
-      # rubocop:enable Lint/UnderscorePrefixedVariableName
 
-      def resolve_tags(tags)
-        if @tags.any? && tags
+      # rubocop:enable Lint/UnderscorePrefixedVariableName
+      def resolve_tags(tags, service)
+        merged_tags = if @tags.any? && tags
           # Combine default tags with provided tags,
           # preferring provided tags.
           @tags.merge(tags)
@@ -441,6 +535,11 @@ module Datadog
           # Use provided tags or default tags if none.
           tags || @tags.dup
         end
+        # Remove version tag if service is not the default service
+        if merged_tags.key?(Core::Environment::Ext::TAG_VERSION) && service && service != @default_service
+          merged_tags.delete(Core::Environment::Ext::TAG_VERSION)
+        end
+        merged_tags
       end
 
       # Manually activate and deactivate the trace, when the span completes.
@@ -478,13 +577,14 @@ module Datadog
       SAMPLE_TRACE_LOG_ONLY_ONCE = Core::Utils::OnlyOnce.new
       private_constant :SAMPLE_TRACE_LOG_ONLY_ONCE
 
+      RECONSIDER_RESOURCE_SAMPLE_TRACE_LOG_ONLY_ONCE = Core::Utils::OnlyOnce.new
+      private_constant :RECONSIDER_RESOURCE_SAMPLE_TRACE_LOG_ONLY_ONCE
+
       def sample_span(trace_op, span)
-        begin
-          @span_sampler.sample!(trace_op, span)
-        rescue StandardError => e
-          SAMPLE_SPAN_LOG_ONLY_ONCE.run do
-            Datadog.logger.warn { "Failed to sample span: #{e.class.name} #{e} at #{Array(e.backtrace).first}" }
-          end
+        @span_sampler.sample!(trace_op, span)
+      rescue => e
+        SAMPLE_SPAN_LOG_ONLY_ONCE.run do
+          logger.warn { "Failed to sample span: #{e.class}: #{e.message} at #{Array(e.backtrace).first}" }
         end
       end
 
@@ -493,14 +593,11 @@ module Datadog
 
       # Flush finished spans from the trace buffer, send them to writer.
       def flush_trace(trace_op)
-        sample_trace(trace_op) unless trace_op.sampling_priority
-        begin
-          trace = @trace_flush.consume!(trace_op)
-          write(trace) if trace && !trace.empty?
-        rescue StandardError => e
-          FLUSH_TRACE_LOG_ONLY_ONCE.run do
-            Datadog.logger.warn { "Failed to flush trace: #{e.class.name} #{e} at #{Array(e.backtrace).first}" }
-          end
+        trace = @trace_flush.consume!(trace_op)
+        write(trace) if trace && !trace.empty?
+      rescue => e
+        FLUSH_TRACE_LOG_ONLY_ONCE.run do
+          logger.warn { "Failed to flush trace: #{e.class}: #{e.message} at #{Array(e.backtrace).first}" }
         end
       end
 
@@ -513,7 +610,7 @@ module Datadog
         return unless trace && @writer
 
         if Datadog.configuration.diagnostics.debug
-          Datadog.logger.debug { "Writing #{trace.length} spans (enabled: #{@enabled})\n#{trace.spans.pretty_inspect}" }
+          logger.debug { "Writing #{trace.length} spans (enabled: #{@enabled})\n#{trace.spans.pretty_inspect}" }
         end
 
         @writer.write(trace)
@@ -522,20 +619,58 @@ module Datadog
 
       # TODO: Make these dummy objects singletons to preserve memory.
       def skip_trace(name)
-        span = SpanOperation.new(name)
+        span = SpanOperation.new(name, logger: logger)
 
         if block_given?
-          trace = TraceOperation.new
+          trace = TraceOperation.new(logger: logger)
           yield(span, trace)
         else
           span
         end
       end
 
+      # Decide whether upstream sampling priority should be propagated, by taking into account
+      # the upstream tags and the configuration.
+      # We should always propagate if APM is enabled.
+      #
+      # e.g.: upstream tags containing dd.p.ts: 02, and appsec is enabled, return true.
+      def propagate_sampling_priority?(upstream_tags:)
+        return true if apm_tracing_enabled
+
+        if upstream_tags&.key?(Tracing::Metadata::Ext::Distributed::TAG_TRACE_SOURCE)
+          appsec_bit = upstream_tags[Tracing::Metadata::Ext::Distributed::TAG_TRACE_SOURCE].to_i(16) &
+            Datadog::AppSec::Ext::PRODUCT_BIT
+          return appsec_enabled if appsec_bit != 0
+
+          ai_guard_bit = upstream_tags[Tracing::Metadata::Ext::Distributed::TAG_TRACE_SOURCE].to_i(16) &
+            Datadog::AIGuard::Ext::PRODUCT_BIT
+          return ai_guard_enabled if ai_guard_bit != 0
+        end
+
+        false
+      end
+
       def profiling_enabled
         @profiling_enabled ||=
           !!(defined?(Datadog::Profiling) && Datadog::Profiling.respond_to?(:enabled?) && Datadog::Profiling.enabled?)
       end
+
+      def appsec_enabled
+        @appsec_enabled ||= Datadog.configuration.appsec.enabled
+      end
+
+      def ai_guard_enabled
+        @ai_guard_enabled ||= Datadog.configuration.respond_to?(:ai_guard) && Datadog.configuration.ai_guard.enabled
+      end
+
+      # Due to APM Tracing (the product) and Tracing (the transport) being intertwined, we cannot completely disabled APM
+      # without also disabling the tracer. When setting `@apm_tracing_enabled` to `false`, it does not disable the tracer,
+      # but rather only sends heartbeat traces (1 per minutes), so that the service is considered alive in the backend.
+      # Other products (like ASM) can then set the sampling priority of their traces to `MANUAL_KEEP`,
+      # effectively allowing standalone products to work without APM.
+      def apm_tracing_enabled
+        @apm_tracing_enabled ||= Datadog.configuration.apm.tracing.enabled
+      end
     end
   end
 end

data/lib/datadog/tracing/transport/http/client.rb

@@ -1,55 +1,42 @@
 # frozen_string_literal: true
 
-require_relative 'statistics'
-require_relative '../../../core/transport/http/env'
-require_relative '../../../core/transport/http/response'
+require_relative "statistics"
+require_relative "../../../core/transport/http/client"
+require_relative "../../../core/transport/http/env"
+require_relative "../../../core/transport/http/response"
 
 module Datadog
   module Tracing
     module Transport
       module HTTP
         # Routes, encodes, and sends tracer data to the trace agent via HTTP.
-        class Client
+        class Client < Core::Transport::HTTP::Client
           include Datadog::Tracing::Transport::HTTP::Statistics
 
-          attr_reader :api
+          private
 
-          def initialize(api)
-            @api = api
-          end
-
-          def send_request(request, &block)
-            # Build request into env
-            env = build_env(request)
-
-            # Get responses from API
-            response = yield(api, env)
+          def on_response(response)
+            super
 
             # Update statistics
             update_stats_from_response!(response)
+          end
 
-            response
-          rescue StandardError => e
-            message =
-              "Internal error during #{self.class.name} request. Cause: #{e.class.name} #{e.message} " \
-                "Location: #{Array(e.backtrace).first}"
+          def on_exception(exception)
+            # Note: this method does NOT call super - it has replacement
+            # logic for how to log the exception.
+
+            message = build_exception_message(exception)
 
-            # Log error
             if stats.consecutive_errors > 0
-              Datadog.logger.debug(message)
+              logger.debug(message)
             else
               # Not to report telemetry logs
-              Datadog.logger.error(message)
+              logger.error(message)
             end
 
             # Update statistics
-            update_stats_from_exception!(e)
-
-            Datadog::Core::Transport::InternalErrorResponse.new(e)
-          end
-
-          def build_env(request)
-            Datadog::Core::Transport::HTTP::Env.new(request)
+            update_stats_from_exception!(exception)
           end
         end
       end