datadog 2.7.1 → 2.18.0

data/lib/datadog/tracing/tracer.rb

@@ -28,7 +28,8 @@ module Datadog
         :provider,
         :sampler,
         :span_sampler,
-        :tags
+        :tags,
+        :logger
 
       attr_accessor \
         :default_service,
@@ -48,21 +49,28 @@ module Datadog
       # @param tags [Hash] default tags added to all spans
       # @param writer [Datadog::Tracing::Writer] consumes traces returned by the provided +trace_flush+
       def initialize(
+        # rubocop:disable Style/KeywordParametersOrder
+        # https://github.com/rubocop/rubocop/issues/13933
         trace_flush: Flush::Finished.new,
         context_provider: DefaultContextProvider.new,
         default_service: Core::Environment::Ext::FALLBACK_SERVICE_NAME,
         enabled: true,
+        logger: Datadog.logger,
         sampler: Sampling::PrioritySampler.new(
           base_sampler: Sampling::AllSampler.new,
           post_sampler: Sampling::RuleSampler.new
         ),
         span_sampler: Sampling::Span::Sampler.new,
         tags: {},
-        writer: Writer.new
+        # writer is not defaulted because creating it requires agent_settings,
+        # which we do not have here and otherwise do not need.
+        writer:
+        # rubocop:enable Style/KeywordParametersOrder
       )
         @trace_flush = trace_flush
         @default_service = default_service
         @enabled = enabled
+        @logger = logger
         @provider = context_provider
         @sampler = sampler
         @span_sampler = span_sampler
@@ -146,7 +154,7 @@ module Datadog
                     active_trace
                   end
         rescue StandardError => e
-          Datadog.logger.debug { "Failed to trace: #{e}" }
+          logger.debug { "Failed to trace: #{e}" }
 
           # Tracing failed: fallback and run code without tracing.
           return skip_trace(name, &block)
@@ -265,10 +273,10 @@ module Datadog
       # Sample a span, tagging the trace as appropriate.
       def sample_trace(trace_op)
         begin
-          @sampler.sample!(trace_op)
+          @sampler.sample!(trace_op) if trace_op.sampling_priority.nil?
         rescue StandardError => e
           SAMPLE_TRACE_LOG_ONLY_ONCE.run do
-            Datadog.logger.warn { "Failed to sample trace: #{e.class.name} #{e} at #{Array(e.backtrace).first}" }
+            logger.warn { "Failed to sample trace: #{e.class.name} #{e} at #{Array(e.backtrace).first}" }
           end
         end
       end
@@ -330,24 +338,32 @@ module Datadog
         hostname = hostname && !hostname.empty? ? hostname : nil
 
         if digest
+          sampling_priority = if propagate_sampling_priority?(upstream_tags: digest.trace_distributed_tags)
+                                digest.trace_sampling_priority
+                              end
           TraceOperation.new(
+            logger: logger,
             hostname: hostname,
             profiling_enabled: profiling_enabled,
+            apm_tracing_enabled: apm_tracing_enabled,
             id: digest.trace_id,
             origin: digest.trace_origin,
             parent_span_id: digest.span_id,
-            sampling_priority: digest.trace_sampling_priority,
+            sampling_priority: sampling_priority,
             # Distributed tags are just regular trace tags with special meaning to Datadog
             tags: digest.trace_distributed_tags,
             trace_state: digest.trace_state,
             trace_state_unknown_fields: digest.trace_state_unknown_fields,
             remote_parent: digest.span_remote,
-            tracer: self
+            tracer: self,
+            baggage: digest.baggage
           )
         else
           TraceOperation.new(
+            logger: logger,
             hostname: hostname,
             profiling_enabled: profiling_enabled,
+            apm_tracing_enabled: apm_tracing_enabled,
             remote_parent: false,
             tracer: self
           )
@@ -362,7 +378,12 @@ module Datadog
           event_span_op.service ||= @default_service
         end
 
+        events.trace_propagated.subscribe do |event_trace_op|
+          sample_trace(event_trace_op)
+        end
+
         events.span_finished.subscribe do |event_span, event_trace_op|
+          sample_trace(trace_op) if event_trace_op.sampling_priority.nil?
           sample_span(event_trace_op, event_span)
           flush_trace(event_trace_op)
         end
@@ -396,12 +417,13 @@ module Datadog
       )
         trace = _trace || start_trace(continue_from: continue_from)
 
-        events = SpanOperation::Events.new
+        events = SpanOperation::Events.new(logger: logger)
 
         if block
           # Ignore start time if a block has been given
           trace.measure(
             name,
+            logger: logger,
             events: events,
             on_error: on_error,
             resource: resource,
@@ -415,6 +437,7 @@ module Datadog
           # Return the new span
           span = trace.build_span(
             name,
+            logger: logger,
             events: events,
             on_error: on_error,
             resource: resource,
@@ -488,7 +511,7 @@ module Datadog
           @span_sampler.sample!(trace_op, span)
         rescue StandardError => e
           SAMPLE_SPAN_LOG_ONLY_ONCE.run do
-            Datadog.logger.warn { "Failed to sample span: #{e.class.name} #{e} at #{Array(e.backtrace).first}" }
+            logger.warn { "Failed to sample span: #{e.class.name} #{e} at #{Array(e.backtrace).first}" }
           end
         end
       end
@@ -498,13 +521,12 @@ module Datadog
 
       # Flush finished spans from the trace buffer, send them to writer.
       def flush_trace(trace_op)
-        sample_trace(trace_op) unless trace_op.sampling_priority
         begin
           trace = @trace_flush.consume!(trace_op)
           write(trace) if trace && !trace.empty?
         rescue StandardError => e
           FLUSH_TRACE_LOG_ONLY_ONCE.run do
-            Datadog.logger.warn { "Failed to flush trace: #{e.class.name} #{e} at #{Array(e.backtrace).first}" }
+            logger.warn { "Failed to flush trace: #{e.class.name} #{e} at #{Array(e.backtrace).first}" }
           end
         end
       end
@@ -518,7 +540,7 @@ module Datadog
         return unless trace && @writer
 
         if Datadog.configuration.diagnostics.debug
-          Datadog.logger.debug { "Writing #{trace.length} spans (enabled: #{@enabled})\n#{trace.spans.pretty_inspect}" }
+          logger.debug { "Writing #{trace.length} spans (enabled: #{@enabled})\n#{trace.spans.pretty_inspect}" }
         end
 
         @writer.write(trace)
@@ -527,20 +549,50 @@ module Datadog
 
       # TODO: Make these dummy objects singletons to preserve memory.
       def skip_trace(name)
-        span = SpanOperation.new(name)
+        span = SpanOperation.new(name, logger: logger)
 
         if block_given?
-          trace = TraceOperation.new
+          trace = TraceOperation.new(logger: logger)
           yield(span, trace)
         else
           span
         end
       end
 
+      # Decide whether upstream sampling priority should be propagated, by taking into account
+      # the upstream tags and the configuration.
+      # We should always propagate if APM is enabled.
+      #
+      # e.g.: upstream tags containing dd.p.ts: 02, and appsec is enabled, return true.
+      def propagate_sampling_priority?(upstream_tags:)
+        return true if apm_tracing_enabled
+
+        if upstream_tags&.key?(Tracing::Metadata::Ext::Distributed::TAG_TRACE_SOURCE)
+          appsec_bit = upstream_tags[Tracing::Metadata::Ext::Distributed::TAG_TRACE_SOURCE].to_i(16) &
+            Datadog::AppSec::Ext::PRODUCT_BIT
+          return appsec_enabled if appsec_bit != 0
+        end
+
+        false
+      end
+
       def profiling_enabled
         @profiling_enabled ||=
           !!(defined?(Datadog::Profiling) && Datadog::Profiling.respond_to?(:enabled?) && Datadog::Profiling.enabled?)
       end
+
+      def appsec_enabled
+        @appsec_enabled ||= Datadog.configuration.appsec.enabled
+      end
+
+      # Due to APM Tracing (the product) and Tracing (the transport) being intertwined, we cannot completely disabled APM
+      # without also disabling the tracer. When setting `@apm_tracing_enabled` to `false`, it does not disable the tracer,
+      # but rather only sends heartbeat traces (1 per minutes), so that the service is considered alive in the backend.
+      # Other products (like ASM) can then set the sampling priority of their traces to `MANUAL_KEEP`,
+      # effectively allowing standalone products to work without APM.
+      def apm_tracing_enabled
+        @apm_tracing_enabled ||= Datadog.configuration.apm.tracing.enabled
+      end
     end
   end
 end

data/lib/datadog/tracing/transport/http/api.rb

@@ -3,7 +3,8 @@
 require_relative '../../../core/encoding'
 
 require_relative '../../../core/transport/http/api/map'
-require_relative 'api/spec'
+require_relative '../../../core/transport/http/api/instance'
+require_relative '../../../core/transport/http/api/spec'
 
 require_relative 'traces'
 
@@ -20,15 +21,15 @@ module Datadog
           module_function
 
           def defaults
-            Datadog::Core::Transport::HTTP::API::Map[
-              V4 => Spec.new do |s|
+            Core::Transport::HTTP::API::Map[
+              V4 => Traces::API::Spec.new do |s|
                 s.traces = Traces::API::Endpoint.new(
                   '/v0.4/traces',
                   Core::Encoding::MsgpackEncoder,
                   service_rates: true
                 )
               end,
-              V3 => Spec.new do |s|
+              V3 => Traces::API::Spec.new do |s|
                 s.traces = Traces::API::Endpoint.new(
                   '/v0.3/traces',
                   Core::Encoding::MsgpackEncoder

data/lib/datadog/tracing/transport/http/client.rb

@@ -12,10 +12,11 @@ module Datadog
         class Client
           include Datadog::Tracing::Transport::HTTP::Statistics
 
-          attr_reader :api
+          attr_reader :api, :logger
 
-          def initialize(api)
+          def initialize(api, logger: Datadog.logger)
             @api = api
+            @logger = logger
           end
 
           def send_request(request, &block)
@@ -36,10 +37,10 @@ module Datadog
 
             # Log error
             if stats.consecutive_errors > 0
-              Datadog.logger.debug(message)
+              logger.debug(message)
             else
               # Not to report telemetry logs
-              Datadog.logger.error(message)
+              logger.error(message)
             end
 
             # Update statistics

data/lib/datadog/tracing/transport/http/traces.rb

@@ -6,7 +6,8 @@ require_relative '../traces'
 require_relative 'client'
 require_relative '../../../core/transport/http/response'
 require_relative '../../../core/transport/http/api/endpoint'
-require_relative 'api/instance'
+require_relative '../../../core/transport/http/api/spec'
+require_relative '../../../core/transport/http/api/instance'
 
 module Datadog
   module Tracing
@@ -36,16 +37,12 @@ module Datadog
           end
 
           module API
-            # Extensions for HTTP API Spec
-            module Spec
-              attr_reader :traces
-
-              def traces=(endpoint)
-                @traces = endpoint
-              end
+            # HTTP API Spec
+            class Spec < Core::Transport::HTTP::API::Spec
+              attr_accessor :traces
 
               def send_traces(env, &block)
-                raise NoTraceEndpointDefinedError, self if traces.nil?
+                raise Core::Transport::HTTP::API::Spec::EndpointNotDefinedError.new('traces', self) if traces.nil?
 
                 traces.call(env, &block)
               end
@@ -53,47 +50,21 @@ module Datadog
               def encoder
                 traces.encoder
               end
-
-              # Raised when traces sent but no traces endpoint is defined
-              class NoTraceEndpointDefinedError < StandardError
-                attr_reader :spec
-
-                def initialize(spec)
-                  super
-
-                  @spec = spec
-                end
-
-                def message
-                  'No trace endpoint is defined for API specification!'
-                end
-              end
             end
 
-            # Extensions for HTTP API Instance
-            module Instance
+            # HTTP API Instance
+            class Instance < Core::Transport::HTTP::API::Instance
               def send_traces(env)
-                raise TracesNotSupportedError, spec unless spec.is_a?(Traces::API::Spec)
+                unless spec.is_a?(Traces::API::Spec)
+                  raise Core::Transport::HTTP::API::Instance::EndpointNotSupportedError.new(
+                    'traces', self
+                  )
+                end
 
                 spec.send_traces(env) do |request_env|
                   call(request_env)
                 end
               end
-
-              # Raised when traces sent to API that does not support traces
-              class TracesNotSupportedError < StandardError
-                attr_reader :spec
-
-                def initialize(spec)
-                  super
-
-                  @spec = spec
-                end
-
-                def message
-                  'Traces not supported for this API!'
-                end
-              end
             end
 
             # Endpoint for submitting trace data
@@ -143,8 +114,6 @@ module Datadog
 
           # Add traces behavior to transport components
           HTTP::Client.include(Traces::Client)
-          HTTP::API::Spec.include(Traces::API::Spec)
-          HTTP::API::Instance.include(Traces::API::Instance)
         end
       end
     end

data/lib/datadog/tracing/transport/http.rb

@@ -1,15 +1,10 @@
 # frozen_string_literal: true
 
-require 'uri'
-
 require_relative '../../core/environment/container'
 require_relative '../../core/environment/ext'
 require_relative '../../core/transport/ext'
-require_relative '../../core/transport/http/adapters/net'
-require_relative '../../core/transport/http/adapters/test'
-require_relative '../../core/transport/http/adapters/unix_socket'
+require_relative '../../core/transport/http'
 require_relative 'http/api'
-require_relative 'http/builder'
 require_relative '../../../datadog/version'
 
 module Datadog
@@ -17,84 +12,32 @@ module Datadog
     module Transport
       # Namespace for HTTP transport components
       module HTTP
-        # NOTE: Due to... legacy reasons... This class likes having a default `AgentSettings` instance to fall back to.
-        # Because we generate this instance with an empty instance of `Settings`, the resulting `AgentSettings` below
-        # represents only settings specified via environment variables + the usual defaults.
-        #
-        # DO NOT USE THIS IN NEW CODE, as it ignores any settings specified by users via `Datadog.configure`.
-        DO_NOT_USE_ENVIRONMENT_AGENT_SETTINGS = Datadog::Core::Configuration::AgentSettingsResolver.call(
-          Datadog::Core::Configuration::Settings.new,
-          logger: nil,
-        )
-
         module_function
 
-        # Builds a new Transport::HTTP::Client
-        def new(&block)
-          Builder.new(&block).to_transport
-        end
-
         # Builds a new Transport::HTTP::Client with default settings
         # Pass a block to override any settings.
         def default(
-          agent_settings: DO_NOT_USE_ENVIRONMENT_AGENT_SETTINGS,
-          **options
+          agent_settings:,
+          logger: Datadog.logger,
+          api_version: nil,
+          headers: nil
         )
-          new do |transport|
-            transport.adapter(agent_settings)
-            transport.headers default_headers
-
+          Core::Transport::HTTP.build(
+            api_instance_class: Traces::API::Instance,
+            agent_settings: agent_settings,
+            logger: logger,
+            api_version: api_version,
+            headers: headers
+          ) do |transport|
             apis = API.defaults
 
             transport.api API::V4, apis[API::V4], fallback: API::V3, default: true
             transport.api API::V3, apis[API::V3]
 
-            # Apply any settings given by options
-            unless options.empty?
-              transport.default_api = options[:api_version] if options.key?(:api_version)
-              transport.headers options[:headers] if options.key?(:headers)
-            end
-
             # Call block to apply any customization, if provided
             yield(transport) if block_given?
-          end
+          end.to_transport(Transport::Traces::Transport)
         end
-
-        def default_headers
-          {
-            Datadog::Core::Transport::Ext::HTTP::HEADER_CLIENT_COMPUTED_TOP_LEVEL => '1',
-            Datadog::Core::Transport::Ext::HTTP::HEADER_META_LANG => Datadog::Core::Environment::Ext::LANG,
-            Datadog::Core::Transport::Ext::HTTP::HEADER_META_LANG_VERSION => Datadog::Core::Environment::Ext::LANG_VERSION,
-            Datadog::Core::Transport::Ext::HTTP::HEADER_META_LANG_INTERPRETER =>
-              Datadog::Core::Environment::Ext::LANG_INTERPRETER,
-            Datadog::Core::Transport::Ext::HTTP::HEADER_META_LANG_INTERPRETER_VENDOR => Core::Environment::Ext::LANG_ENGINE,
-            Datadog::Core::Transport::Ext::HTTP::HEADER_META_TRACER_VERSION =>
-              Datadog::Core::Environment::Ext::GEM_DATADOG_VERSION
-          }.tap do |headers|
-            # Add container ID, if present.
-            container_id = Datadog::Core::Environment::Container.container_id
-            headers[Datadog::Core::Transport::Ext::HTTP::HEADER_CONTAINER_ID] = container_id unless container_id.nil?
-            # Pretend that stats computation are already done by the client
-            if Datadog.configuration.appsec.standalone.enabled
-              headers[Datadog::Core::Transport::Ext::HTTP::HEADER_CLIENT_COMPUTED_STATS] = 'yes'
-            end
-          end
-        end
-
-        def default_adapter
-          Datadog::Core::Configuration::Ext::Agent::HTTP::ADAPTER
-        end
-
-        # Add adapters to registry
-        Builder::REGISTRY.set(
-          Datadog::Core::Transport::HTTP::Adapters::Net,
-          Datadog::Core::Configuration::Ext::Agent::HTTP::ADAPTER
-        )
-        Builder::REGISTRY.set(Datadog::Core::Transport::HTTP::Adapters::Test, Datadog::Core::Transport::Ext::Test::ADAPTER)
-        Builder::REGISTRY.set(
-          Datadog::Core::Transport::HTTP::Adapters::UnixSocket,
-          Datadog::Core::Transport::Ext::UnixSocket::ADAPTER
-        )
       end
     end
   end

data/lib/datadog/tracing/transport/serializable_trace.rb

@@ -12,8 +12,11 @@ module Datadog
         attr_reader \
           :trace
 
-        def initialize(trace)
+        # @param trace [Datadog::Trace] the trace to serialize
+        # @param native_events_supported [Boolean] whether the agent supports span events as a top-level field
+        def initialize(trace, native_events_supported:)
           @trace = trace
+          @native_events_supported = native_events_supported
         end
 
         # MessagePack serializer interface. Making this object
@@ -26,13 +29,17 @@ module Datadog
         # @param packer [MessagePack::Packer] serialization buffer, can be +nil+ with JRuby
         def to_msgpack(packer = nil)
           # As of 1.3.3, JRuby implementation doesn't pass an existing packer
-          trace.spans.map { |s| SerializableSpan.new(s) }.to_msgpack(packer)
+          trace.spans.map do |s|
+            SerializableSpan.new(s, native_events_supported: @native_events_supported)
+          end.to_msgpack(packer)
         end
 
         # JSON serializer interface.
         # Used by older version of the transport.
         def to_json(*args)
-          trace.spans.map { |s| SerializableSpan.new(s).to_hash }.to_json(*args)
+          trace.spans.map do |s|
+            SerializableSpan.new(s, native_events_supported: @native_events_supported).to_hash
+          end.to_json(*args)
         end
       end
 
@@ -41,9 +48,12 @@ module Datadog
         attr_reader \
           :span
 
-        def initialize(span)
+        # @param span [Datadog::Span] the span to serialize
+        # @param native_events_supported [Boolean] whether the agent supports span events as a top-level field
+        def initialize(span, native_events_supported:)
           @span = span
           @trace_id = Tracing::Utils::TraceId.to_low_order(span.trace_id)
+          @native_events_supported = native_events_supported
         end
 
         # MessagePack serializer interface. Making this object
@@ -55,10 +65,13 @@ module Datadog
         #
         # @param packer [MessagePack::Packer] serialization buffer, can be +nil+ with JRuby
         # rubocop:disable Metrics/AbcSize
+        # rubocop:disable Metrics/MethodLength
         def to_msgpack(packer = nil)
           packer ||= MessagePack::Packer.new
 
-          number_of_elements_to_write = 11
+          number_of_elements_to_write = 12
+
+          number_of_elements_to_write += 1 if span.events.any? && @native_events_supported
 
           if span.stopped?
             packer.write_map_header(number_of_elements_to_write + 2) # Set header with how many elements in the map
@@ -72,8 +85,16 @@ module Datadog
             packer.write_map_header(number_of_elements_to_write) # Set header with how many elements in the map
           end
 
-          # serialize span events as meta tags
-          span.set_tag('events', span.events.map(&:to_hash).to_json) if span.events.any?
+          if span.events.any?
+            if @native_events_supported
+              # Use top-level field for native events
+              packer.write('span_events')
+              packer.write(span.events.map(&:to_native_format))
+            else
+              # Serialize span events as meta tags
+              span.set_tag('events', span.events.map(&:to_hash).to_json)
+            end
+          end
 
           # DEV: We use strings as keys here, instead of symbols, as
           # DEV: MessagePack will ultimately convert them to strings.
@@ -96,6 +117,8 @@ module Datadog
           packer.write(span.meta)
           packer.write('metrics')
           packer.write(span.metrics)
+          packer.write('meta_struct')
+          packer.write(span.metastruct)
           packer.write('span_links')
           packer.write(span.links.map(&:to_hash))
           packer.write('error')
@@ -103,6 +126,7 @@ module Datadog
           packer
         end
         # rubocop:enable Metrics/AbcSize
+        # rubocop:enable Metrics/MethodLength
 
         # JSON serializer interface.
         # Used by older version of the transport.

data/lib/datadog/tracing/transport/trace_formatter.rb

@@ -59,6 +59,7 @@ module Datadog
           tag_high_order_trace_id!
           tag_sampling_priority!
           tag_profiling_enabled!
+          tag_apm_tracing_disabled!
 
           if first_span
             tag_git_repository_url!
@@ -196,6 +197,12 @@ module Datadog
           )
         end
 
+        def tag_apm_tracing_disabled!
+          return if trace.apm_tracing_enabled
+
+          root_span.set_tag(Tracing::Metadata::Ext::TAG_APM_ENABLED, 0)
+        end
+
         def tag_git_repository_url!
           return if git_repository_url.nil?