datadog 2.8.0 → 2.39.0

data/lib/datadog/tracing/tracer.rb

@@ -1,20 +1,20 @@
 # frozen_string_literal: true
 
-require_relative '../core/environment/ext'
-require_relative '../core/environment/socket'
-
-require_relative 'correlation'
-require_relative 'event'
-require_relative 'flush'
-require_relative 'context_provider'
-require_relative 'sampling/all_sampler'
-require_relative 'sampling/rule_sampler'
-require_relative 'sampling/priority_sampler'
-require_relative 'sampling/span/sampler'
-require_relative 'span_operation'
-require_relative 'trace_digest'
-require_relative 'trace_operation'
-require_relative 'writer'
+require_relative "../core/environment/ext"
+require_relative "../core/environment/socket"
+
+require_relative "correlation"
+require_relative "event"
+require_relative "flush"
+require_relative "context_provider"
+require_relative "sampling/all_sampler"
+require_relative "sampling/rule_sampler"
+require_relative "sampling/priority_sampler"
+require_relative "sampling/span/sampler"
+require_relative "span_operation"
+require_relative "trace_digest"
+require_relative "trace_operation"
+require_relative "writer"
 
 module Datadog
   module Tracing
@@ -28,7 +28,8 @@ module Datadog
         :provider,
         :sampler,
         :span_sampler,
-        :tags
+        :tags,
+        :logger
 
       attr_accessor \
         :default_service,
@@ -48,21 +49,28 @@ module Datadog
       # @param tags [Hash] default tags added to all spans
       # @param writer [Datadog::Tracing::Writer] consumes traces returned by the provided +trace_flush+
       def initialize(
+        # rubocop:disable Style/KeywordParametersOrder
+        # https://github.com/rubocop/rubocop/issues/13933
         trace_flush: Flush::Finished.new,
         context_provider: DefaultContextProvider.new,
         default_service: Core::Environment::Ext::FALLBACK_SERVICE_NAME,
         enabled: true,
+        logger: Datadog.logger,
         sampler: Sampling::PrioritySampler.new(
           base_sampler: Sampling::AllSampler.new,
           post_sampler: Sampling::RuleSampler.new
         ),
         span_sampler: Sampling::Span::Sampler.new,
         tags: {},
-        writer: Writer.new
+        # writer is not defaulted because creating it requires agent_settings,
+        # which we do not have here and otherwise do not need.
+        writer:
+        # rubocop:enable Style/KeywordParametersOrder
       )
         @trace_flush = trace_flush
         @default_service = default_service
         @enabled = enabled
+        @logger = logger
         @provider = context_provider
         @sampler = sampler
         @span_sampler = span_sampler
@@ -121,7 +129,6 @@ module Datadog
       # @yield Optional block where new newly created {Datadog::Tracing::SpanOperation} captures the execution.
       # @yieldparam [Datadog::Tracing::SpanOperation] span_op the newly created and active [Datadog::Tracing::SpanOperation]
       # @yieldparam [Datadog::Tracing::TraceOperation] trace_op the active [Datadog::Tracing::TraceOperation]
-      # rubocop:disable Metrics/MethodLength
       def trace(
         name,
         continue_from: nil,
@@ -141,12 +148,12 @@ module Datadog
           context = call_context
           active_trace = context.active_trace
           trace = if continue_from || active_trace.nil?
-                    start_trace(continue_from: continue_from)
-                  else
-                    active_trace
-                  end
-        rescue StandardError => e
-          Datadog.logger.debug { "Failed to trace: #{e}" }
+            start_trace(continue_from: continue_from)
+          else
+            active_trace
+          end
+        rescue => e
+          logger.debug { "Failed to trace: #{e.class}: #{e.message}" }
 
           # Tracing failed: fallback and run code without tracing.
           return skip_trace(name, &block)
@@ -186,7 +193,6 @@ module Datadog
           )
         end
       end
-      # rubocop:enable Metrics/MethodLength
 
       # Set the given key / value tag pair at the tracer level. These tags will be
       # appended to each span created by the tracer. Keys and values must be strings.
@@ -217,7 +223,7 @@ module Datadog
       # @return [nil] if no trace is active, and thus no span is active
       def active_span(key = nil)
         trace = active_trace(key)
-        trace.active_span if trace
+        trace&.active_span
       end
 
       # Information about the currently active trace.
@@ -234,10 +240,21 @@ module Datadog
         trace.to_correlation
       end
 
-      # Setup a new trace to continue from where another
+      # Setup a new trace execution context to continue from where another
       # trace left off.
+      # This is useful to continue distributed or async traces.
+      #
+      # The first span created in the restored context is a direct child of the
+      # active span from when the {Datadog::Tracing::TraceDigest} was created.
+      #
+      # When no block is given, the trace context is restored in the current thread.
+      # It remains active until the first span created in this restored context is finished.
+      # After that, if a new span is created, it start a new, unrelated trace.
       #
-      # Used to continue distributed or async traces.
+      # When a block is given, the trace context is restored inside the block execution.
+      # It remains active until the block ends, even when the first span created inside
+      # the block finishes. This means that multiple spans can be direct children of the
+      # active span from when the {Datadog::Tracing::TraceDigest} was created.
       #
       # @param [Datadog::Tracing::TraceDigest] digest continue from the {Datadog::Tracing::TraceDigest}.
       # @param [Thread] key Thread to retrieve trace from. Defaults to current thread. For internal use only.
@@ -253,22 +270,52 @@ module Datadog
         # Start a new trace from the digest
         context = call_context(key)
         original_trace = active_trace(key)
-        trace = start_trace(continue_from: digest)
+        # When we want the trace to be bound to a block, we cannot let
+        # it auto finish when the local root span finishes. This would
+        # create mutiple traces inside the block. Instead, we'll
+        # expliclity finish the trace after the block finishes.
+        auto_finish = !block
+
+        trace = start_trace(continue_from: digest, auto_finish: auto_finish)
 
         # If block hasn't been given; we need to manually deactivate
         # this trace. Subscribe to the trace finished event to do this.
         subscribe_trace_deactivation!(context, trace, original_trace) unless block
 
-        context.activate!(trace, &block)
+        if block
+          # When a block is given, the trace will be active until the block finishes.
+          context.activate!(trace) do
+            yield
+          ensure # We have to flush even when an error occurs
+            # On block completion, force the trace to finish and flush its finished spans.
+            # Unfinished spans are lost as the {TraceOperation} has ended.
+            trace.finish!
+            flush_trace(trace)
+          end
+        else
+          # Otherwise, the trace will be bound to the current thread after this point
+          context.activate!(trace)
+        end
       end
 
       # Sample a span, tagging the trace as appropriate.
       def sample_trace(trace_op)
-        begin
-          @sampler.sample!(trace_op)
-        rescue StandardError => e
-          SAMPLE_TRACE_LOG_ONLY_ONCE.run do
-            Datadog.logger.warn { "Failed to sample trace: #{e.class.name} #{e} at #{Array(e.backtrace).first}" }
+        @sampler.sample!(trace_op) if trace_op.sampling_priority.nil?
+      rescue => e
+        SAMPLE_TRACE_LOG_ONLY_ONCE.run do
+          logger.warn { "Failed to sample trace: #{e.class}: #{e.message} at #{Array(e.backtrace).first}" }
+        end
+      end
+
+      def reconsider_trace_sampling_on_resource(trace_op)
+        return unless trace_op.reconsider_resource_sample?
+        return unless @sampler.respond_to?(:reconsider_sample_resource!)
+
+        @sampler.reconsider_sample_resource!(trace_op)
+      rescue => e
+        RECONSIDER_RESOURCE_SAMPLE_TRACE_LOG_ONLY_ONCE.run do
+          logger.warn do
+            "Failed to reconsider trace sampling: #{e.class}: #{e.message} at #{Array(e.backtrace).first}"
           end
         end
       end
@@ -289,7 +336,7 @@ module Datadog
         #       description of and constraints on arguments.
         # rubocop:disable Lint/UselessMethodDefinition
         def publish(trace)
-          super(trace)
+          super
         end
         # rubocop:enable Lint/UselessMethodDefinition
       end
@@ -307,7 +354,7 @@ module Datadog
       def shutdown!
         return unless @enabled
 
-        @writer.stop if @writer
+        @writer&.stop
       end
 
       private
@@ -324,32 +371,41 @@ module Datadog
         @provider.context(key)
       end
 
-      def build_trace(digest = nil)
-        # Resolve hostname if configured
-        hostname = Core::Environment::Socket.hostname if Datadog.configuration.tracing.report_hostname
-        hostname = hostname && !hostname.empty? ? hostname : nil
+      def build_trace(digest, auto_finish)
+        hostname = Core::Environment::Socket.resolved_hostname(Datadog.configuration)
 
         if digest
+          sampling_priority = if propagate_sampling_priority?(upstream_tags: digest.trace_distributed_tags)
+            digest.trace_sampling_priority
+          end
           TraceOperation.new(
+            logger: logger,
             hostname: hostname,
             profiling_enabled: profiling_enabled,
+            apm_tracing_enabled: apm_tracing_enabled,
             id: digest.trace_id,
             origin: digest.trace_origin,
             parent_span_id: digest.span_id,
-            sampling_priority: digest.trace_sampling_priority,
+            sampling_priority: sampling_priority,
+            span_links: digest.span_links,
             # Distributed tags are just regular trace tags with special meaning to Datadog
             tags: digest.trace_distributed_tags,
             trace_state: digest.trace_state,
             trace_state_unknown_fields: digest.trace_state_unknown_fields,
             remote_parent: digest.span_remote,
-            tracer: self
+            tracer: self,
+            baggage: digest.baggage,
+            auto_finish: auto_finish
           )
         else
           TraceOperation.new(
+            logger: logger,
             hostname: hostname,
             profiling_enabled: profiling_enabled,
+            apm_tracing_enabled: apm_tracing_enabled,
             remote_parent: false,
-            tracer: self
+            tracer: self,
+            auto_finish: auto_finish
           )
         end
       end
@@ -362,17 +418,40 @@ module Datadog
           event_span_op.service ||= @default_service
         end
 
+        events.span_before_finish.subscribe do |event_span_op, _event_trace_op|
+          if event_span_op.service && event_span_op.service != @default_service
+            event_span_op.set_tag(Tracing::Metadata::Ext::TAG_BASE_SERVICE, @default_service)
+            event_span_op.set_tag(Tracing::Metadata::Ext::TAG_SVC_SRC, Tracing::Metadata::Ext::SVC_SRC_MANUAL) unless event_span_op.get_tag(Tracing::Metadata::Ext::TAG_SVC_SRC)
+          else
+            event_span_op.send(:meta).delete(Tracing::Metadata::Ext::TAG_SVC_SRC)
+          end
+        end
+
+        events.trace_propagated.subscribe do |event_trace_op|
+          sample_trace(event_trace_op)
+        end
+
         events.span_finished.subscribe do |event_span, event_trace_op|
+          sample_trace(trace_op) if event_trace_op.sampling_priority.nil?
           sample_span(event_trace_op, event_span)
           flush_trace(event_trace_op)
         end
+
+        # Conditionally subscribe to the less common sampling rules below, to avoid
+        # measurable unnecessary performance overhead when they are not present.
+
+        if @sampler.respond_to?(:resource_sampling?) && @sampler.resource_sampling?
+          events.trace_resource_change.subscribe do |event_trace_op|
+            reconsider_trace_sampling_on_resource(event_trace_op)
+          end
+        end
       end
 
       # Creates a new TraceOperation, with events bounds to this Tracer instance.
       # @return [TraceOperation]
-      def start_trace(continue_from: nil)
+      def start_trace(continue_from: nil, auto_finish: true)
         # Build a new trace using digest if provided.
-        trace = build_trace(continue_from)
+        trace = build_trace(continue_from, auto_finish)
 
         # Bind trace events: sample trace, set default service, flush spans.
         bind_trace_events!(trace)
@@ -396,12 +475,13 @@ module Datadog
       )
         trace = _trace || start_trace(continue_from: continue_from)
 
-        events = SpanOperation::Events.new
+        events = SpanOperation::Events.new(logger: logger)
 
         if block
           # Ignore start time if a block has been given
           trace.measure(
             name,
+            logger: logger,
             events: events,
             on_error: on_error,
             resource: resource,
@@ -415,6 +495,7 @@ module Datadog
           # Return the new span
           span = trace.build_span(
             name,
+            logger: logger,
             events: events,
             on_error: on_error,
             resource: resource,
@@ -430,17 +511,17 @@ module Datadog
           span
         end
       end
-      # rubocop:enable Lint/UnderscorePrefixedVariableName
 
+      # rubocop:enable Lint/UnderscorePrefixedVariableName
       def resolve_tags(tags, service)
         merged_tags = if @tags.any? && tags
-                        # Combine default tags with provided tags,
-                        # preferring provided tags.
-                        @tags.merge(tags)
-                      else
-                        # Use provided tags or default tags if none.
-                        tags || @tags.dup
-                      end
+          # Combine default tags with provided tags,
+          # preferring provided tags.
+          @tags.merge(tags)
+        else
+          # Use provided tags or default tags if none.
+          tags || @tags.dup
+        end
         # Remove version tag if service is not the default service
         if merged_tags.key?(Core::Environment::Ext::TAG_VERSION) && service && service != @default_service
           merged_tags.delete(Core::Environment::Ext::TAG_VERSION)
@@ -483,13 +564,14 @@ module Datadog
       SAMPLE_TRACE_LOG_ONLY_ONCE = Core::Utils::OnlyOnce.new
       private_constant :SAMPLE_TRACE_LOG_ONLY_ONCE
 
+      RECONSIDER_RESOURCE_SAMPLE_TRACE_LOG_ONLY_ONCE = Core::Utils::OnlyOnce.new
+      private_constant :RECONSIDER_RESOURCE_SAMPLE_TRACE_LOG_ONLY_ONCE
+
       def sample_span(trace_op, span)
-        begin
-          @span_sampler.sample!(trace_op, span)
-        rescue StandardError => e
-          SAMPLE_SPAN_LOG_ONLY_ONCE.run do
-            Datadog.logger.warn { "Failed to sample span: #{e.class.name} #{e} at #{Array(e.backtrace).first}" }
-          end
+        @span_sampler.sample!(trace_op, span)
+      rescue => e
+        SAMPLE_SPAN_LOG_ONLY_ONCE.run do
+          logger.warn { "Failed to sample span: #{e.class}: #{e.message} at #{Array(e.backtrace).first}" }
         end
       end
 
@@ -498,14 +580,11 @@ module Datadog
 
       # Flush finished spans from the trace buffer, send them to writer.
       def flush_trace(trace_op)
-        sample_trace(trace_op) unless trace_op.sampling_priority
-        begin
-          trace = @trace_flush.consume!(trace_op)
-          write(trace) if trace && !trace.empty?
-        rescue StandardError => e
-          FLUSH_TRACE_LOG_ONLY_ONCE.run do
-            Datadog.logger.warn { "Failed to flush trace: #{e.class.name} #{e} at #{Array(e.backtrace).first}" }
-          end
+        trace = @trace_flush.consume!(trace_op)
+        write(trace) if trace && !trace.empty?
+      rescue => e
+        FLUSH_TRACE_LOG_ONLY_ONCE.run do
+          logger.warn { "Failed to flush trace: #{e.class}: #{e.message} at #{Array(e.backtrace).first}" }
         end
       end
 
@@ -518,7 +597,7 @@ module Datadog
         return unless trace && @writer
 
         if Datadog.configuration.diagnostics.debug
-          Datadog.logger.debug { "Writing #{trace.length} spans (enabled: #{@enabled})\n#{trace.spans.pretty_inspect}" }
+          logger.debug { "Writing #{trace.length} spans (enabled: #{@enabled})\n#{trace.spans.pretty_inspect}" }
         end
 
         @writer.write(trace)
@@ -527,20 +606,50 @@ module Datadog
 
       # TODO: Make these dummy objects singletons to preserve memory.
       def skip_trace(name)
-        span = SpanOperation.new(name)
+        span = SpanOperation.new(name, logger: logger)
 
         if block_given?
-          trace = TraceOperation.new
+          trace = TraceOperation.new(logger: logger)
           yield(span, trace)
         else
           span
         end
       end
 
+      # Decide whether upstream sampling priority should be propagated, by taking into account
+      # the upstream tags and the configuration.
+      # We should always propagate if APM is enabled.
+      #
+      # e.g.: upstream tags containing dd.p.ts: 02, and appsec is enabled, return true.
+      def propagate_sampling_priority?(upstream_tags:)
+        return true if apm_tracing_enabled
+
+        if upstream_tags&.key?(Tracing::Metadata::Ext::Distributed::TAG_TRACE_SOURCE)
+          appsec_bit = upstream_tags[Tracing::Metadata::Ext::Distributed::TAG_TRACE_SOURCE].to_i(16) &
+            Datadog::AppSec::Ext::PRODUCT_BIT
+          return appsec_enabled if appsec_bit != 0
+        end
+
+        false
+      end
+
       def profiling_enabled
         @profiling_enabled ||=
           !!(defined?(Datadog::Profiling) && Datadog::Profiling.respond_to?(:enabled?) && Datadog::Profiling.enabled?)
       end
+
+      def appsec_enabled
+        @appsec_enabled ||= Datadog.configuration.appsec.enabled
+      end
+
+      # Due to APM Tracing (the product) and Tracing (the transport) being intertwined, we cannot completely disabled APM
+      # without also disabling the tracer. When setting `@apm_tracing_enabled` to `false`, it does not disable the tracer,
+      # but rather only sends heartbeat traces (1 per minutes), so that the service is considered alive in the backend.
+      # Other products (like ASM) can then set the sampling priority of their traces to `MANUAL_KEEP`,
+      # effectively allowing standalone products to work without APM.
+      def apm_tracing_enabled
+        @apm_tracing_enabled ||= Datadog.configuration.apm.tracing.enabled
+      end
     end
   end
 end

data/lib/datadog/tracing/transport/http/client.rb

@@ -1,55 +1,42 @@
 # frozen_string_literal: true
 
-require_relative 'statistics'
-require_relative '../../../core/transport/http/env'
-require_relative '../../../core/transport/http/response'
+require_relative "statistics"
+require_relative "../../../core/transport/http/client"
+require_relative "../../../core/transport/http/env"
+require_relative "../../../core/transport/http/response"
 
 module Datadog
   module Tracing
     module Transport
       module HTTP
         # Routes, encodes, and sends tracer data to the trace agent via HTTP.
-        class Client
+        class Client < Core::Transport::HTTP::Client
           include Datadog::Tracing::Transport::HTTP::Statistics
 
-          attr_reader :api
+          private
 
-          def initialize(api)
-            @api = api
-          end
-
-          def send_request(request, &block)
-            # Build request into env
-            env = build_env(request)
-
-            # Get responses from API
-            response = yield(api, env)
+          def on_response(response)
+            super
 
             # Update statistics
             update_stats_from_response!(response)
+          end
 
-            response
-          rescue StandardError => e
-            message =
-              "Internal error during #{self.class.name} request. Cause: #{e.class.name} #{e.message} " \
-                "Location: #{Array(e.backtrace).first}"
+          def on_exception(exception)
+            # Note: this method does NOT call super - it has replacement
+            # logic for how to log the exception.
+
+            message = build_exception_message(exception)
 
-            # Log error
             if stats.consecutive_errors > 0
-              Datadog.logger.debug(message)
+              logger.debug(message)
             else
               # Not to report telemetry logs
-              Datadog.logger.error(message)
+              logger.error(message)
             end
 
             # Update statistics
-            update_stats_from_exception!(e)
-
-            Datadog::Core::Transport::InternalErrorResponse.new(e)
-          end
-
-          def build_env(request)
-            Datadog::Core::Transport::HTTP::Env.new(request)
+            update_stats_from_exception!(exception)
           end
         end
       end

data/lib/datadog/tracing/transport/http/statistics.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative '../statistics'
+require_relative "../statistics"
 
 module Datadog
   module Tracing
@@ -30,7 +30,7 @@ module Datadog
             private
 
             # The most common status code on a healthy tracer
-            STATUS_CODE_200 = 'status_code:200'
+            STATUS_CODE_200 = "status_code:200"
 
             def metrics_tag_value(status_code)
               if status_code == 200

data/lib/datadog/tracing/transport/http/traces.rb

@@ -1,12 +1,11 @@
 # frozen_string_literal: true
 
-require 'json'
+require "json"
 
-require_relative '../traces'
-require_relative 'client'
-require_relative '../../../core/transport/http/response'
-require_relative '../../../core/transport/http/api/endpoint'
-require_relative 'api/instance'
+require_relative "../../../core/transport/http/response"
+require_relative "../../../core/transport/http/api/endpoint"
+require_relative "../../../core/transport/http/api/instance"
+require_relative "../traces"
 
 module Datadog
   module Tracing
@@ -26,88 +25,15 @@ module Datadog
             end
           end
 
-          # Extensions for HTTP client
-          module Client
-            def send_traces_payload(request)
-              send_request(request) do |api, env|
-                api.send_traces(env)
-              end
-            end
-          end
-
           module API
-            # Extensions for HTTP API Spec
-            module Spec
-              attr_reader :traces
-
-              def traces=(endpoint)
-                @traces = endpoint
-              end
-
-              def send_traces(env, &block)
-                raise NoTraceEndpointDefinedError, self if traces.nil?
-
-                traces.call(env, &block)
-              end
-
-              def encoder
-                traces.encoder
-              end
-
-              # Raised when traces sent but no traces endpoint is defined
-              class NoTraceEndpointDefinedError < StandardError
-                attr_reader :spec
-
-                def initialize(spec)
-                  super
-
-                  @spec = spec
-                end
-
-                def message
-                  'No trace endpoint is defined for API specification!'
-                end
-              end
-            end
-
-            # Extensions for HTTP API Instance
-            module Instance
-              def send_traces(env)
-                raise TracesNotSupportedError, spec unless spec.is_a?(Traces::API::Spec)
-
-                spec.send_traces(env) do |request_env|
-                  call(request_env)
-                end
-              end
-
-              # Raised when traces sent to API that does not support traces
-              class TracesNotSupportedError < StandardError
-                attr_reader :spec
-
-                def initialize(spec)
-                  super
-
-                  @spec = spec
-                end
-
-                def message
-                  'Traces not supported for this API!'
-                end
-              end
-            end
-
             # Endpoint for submitting trace data
             class Endpoint < Datadog::Core::Transport::HTTP::API::Endpoint
-              HEADER_CONTENT_TYPE = 'Content-Type'
-              HEADER_TRACE_COUNT = 'X-Datadog-Trace-Count'
-              SERVICE_RATE_KEY = 'rate_by_service'
-
-              attr_reader \
-                :encoder
+              HEADER_CONTENT_TYPE = "Content-Type"
+              HEADER_TRACE_COUNT = "X-Datadog-Trace-Count"
+              SERVICE_RATE_KEY = "rate_by_service"
 
               def initialize(path, encoder, options = {})
-                super(:post, path)
-                @encoder = encoder
+                super(:post, path, encoder: encoder)
                 @service_rates = options.fetch(:service_rates, false)
               end
 
@@ -124,10 +50,10 @@ module Datadog
                 env.body = env.request.parcel.data
 
                 # Query for response
-                http_response = super(env, &block)
+                http_response = super
 
                 # Process the response
-                response_options = { trace_count: env.request.parcel.trace_count }.tap do |options|
+                response_options = {trace_count: env.request.parcel.trace_count}.tap do |options|
                   # Parse service rates, if configured to do so.
                   if service_rates? && !http_response.payload.to_s.empty?
                     body = JSON.parse(http_response.payload)
@@ -140,11 +66,6 @@ module Datadog
               end
             end
           end
-
-          # Add traces behavior to transport components
-          HTTP::Client.include(Traces::Client)
-          HTTP::API::Spec.include(Traces::API::Spec)
-          HTTP::API::Instance.include(Traces::API::Instance)
         end
       end
     end