datadog 2.4.0 → 2.44.0

data/lib/datadog/tracing/transport/trace_formatter.rb

@@ -1,12 +1,13 @@
 # frozen_string_literal: true
 
-require_relative '../../core/environment/identity'
-require_relative '../../core/environment/socket'
-require_relative '../../core/environment/git'
-require_relative '../../core/git/ext'
-require_relative '../../core/runtime/ext'
-require_relative '../metadata/ext'
-require_relative '../trace_segment'
+require_relative "../../core/environment/identity"
+require_relative "../../core/environment/process"
+require_relative "../../core/environment/socket"
+require_relative "../../core/environment/git"
+require_relative "../../core/git/ext"
+require_relative "../../core/runtime/ext"
+require_relative "../metadata/ext"
+require_relative "../trace_segment"
 
 module Datadog
   module Tracing
@@ -18,8 +19,8 @@ module Datadog
           :first_span,
           :trace
 
-        def self.format!(trace)
-          new(trace).format!
+        def self.format!(trace, transport: nil)
+          new(trace).format!(transport: transport)
         end
 
         def initialize(trace)
@@ -31,7 +32,7 @@ module Datadog
         end
 
         # Modifies a trace so suitable for transport
-        def format!
+        def format!(transport: nil)
           return unless trace
           return trace unless root_span
 
@@ -48,6 +49,7 @@ module Datadog
 
           tag_agent_sample_rate!
           tag_hostname!
+          tag_knuth_sampling_rate!
           tag_lang!
           tag_origin!
           tag_process_id!
@@ -59,12 +61,17 @@ module Datadog
           tag_high_order_trace_id!
           tag_sampling_priority!
           tag_profiling_enabled!
+          tag_apm_tracing_disabled!
 
           if first_span
+            tag_process_tags!
+            tag_sdk_otlp_export!
             tag_git_repository_url!
             tag_git_commit_sha!
           end
 
+          root_span.set_tag("_dd.tracing.transport", transport) if transport && !partial?
+
           trace
         end
 
@@ -107,6 +114,16 @@ module Datadog
           )
         end
 
+        def tag_knuth_sampling_rate!
+          rate = trace.rule_sample_rate || trace.agent_sample_rate
+          return unless rate
+
+          root_span.set_tag(
+            Tracing::Metadata::Ext::Distributed::TAG_KNUTH_SAMPLING_RATE,
+            format("%.6f", (rate * 1e6).round / 1e6).sub(/\.?0+\z/, "")
+          )
+        end
+
         def tag_lang!
           return if trace.lang.nil?
 
@@ -196,6 +213,12 @@ module Datadog
           )
         end
 
+        def tag_apm_tracing_disabled!
+          return if trace.apm_tracing_enabled
+
+          trace.spans.each { |span| span.set_metric(Tracing::Metadata::Ext::TAG_APM_ENABLED, 0) }
+        end
+
         def tag_git_repository_url!
           return if git_repository_url.nil?
 
@@ -208,6 +231,22 @@ module Datadog
           first_span.set_tag(Core::Git::Ext::TAG_COMMIT_SHA, git_commit_sha)
         end
 
+        def tag_process_tags!
+          return unless Datadog.configuration.experimental_propagate_process_tags_enabled
+
+          first_span.set_tag(
+            Core::Environment::Ext::TAG_PROCESS_TAGS,
+            Core::Environment::Process.serialized
+          )
+        end
+
+        # TODO: Revisit when OTLP trace export support lands. The OTLP path sets
+        # `_dd.sdk.otlp_export: "true"` on the OTLP resource, so this tag must not
+        # also be set to "false" on the first span of a chunk exported over OTLP.
+        def tag_sdk_otlp_export!
+          first_span.set_tag(Tracing::Metadata::Ext::TAG_SDK_OTLP_EXPORT, "false")
+        end
+
         private
 
         def partial?

data/lib/datadog/tracing/transport/traces.rb

@@ -1,29 +1,27 @@
 # frozen_string_literal: true
 
-require_relative '../../core/chunker'
-require_relative '../../core/transport/parcel'
-require_relative '../../core/transport/request'
-require_relative 'serializable_trace'
-require_relative 'trace_formatter'
+require_relative "../../core/chunker"
+require_relative "../../core/transport/parcel"
+require_relative "../../core/transport/request"
+require_relative "../../core/transport/transport"
+require_relative "../../core/utils/enumerable_compat"
+require_relative "http/client"
+require_relative "serializable_trace"
+require_relative "span_events_negotiation"
+require_relative "trace_formatter"
 
 module Datadog
   module Tracing
     module Transport
       module Traces
         # Data transfer object for encoded traces
-        class EncodedParcel
-          include Datadog::Core::Transport::Parcel
-
+        class Parcel < Core::Transport::Parcel
           attr_reader :trace_count
 
-          def initialize(data, trace_count)
-            super(data)
+          def initialize(data, trace_count:, **opts)
+            super(data, **opts)
             @trace_count = trace_count
           end
-
-          def count
-            data.length
-          end
         end
 
         # Traces request
@@ -43,15 +41,18 @@ module Datadog
           # We set the value to a conservative 5 MiB, in case network speed is slow.
           DEFAULT_MAX_PAYLOAD_SIZE = 5 * 1024 * 1024
 
-          attr_reader :encoder, :max_size
+          attr_reader :encoder, :max_size, :logger
 
           #
           # Single traces larger than +max_size+ will be discarded.
           #
           # @param encoder [Datadog::Core::Encoding::Encoder]
+          # @param logger [Datadog::Core::Logger]
           # @param max_size [String] maximum acceptable payload size
-          def initialize(encoder, max_size: DEFAULT_MAX_PAYLOAD_SIZE)
+          def initialize(encoder, logger:, native_events_supported:, max_size: DEFAULT_MAX_PAYLOAD_SIZE)
             @encoder = encoder
+            @logger = logger
+            @native_events_supported = native_events_supported
             @max_size = max_size
           end
 
@@ -62,12 +63,9 @@ module Datadog
           # @return [Enumerable[Array[Bytes,Integer]]] list of encoded chunks: each containing a byte array and
           #   number of traces
           def encode_in_chunks(traces)
-            encoded_traces = if traces.respond_to?(:filter_map)
-                               # DEV Supported since Ruby 2.7, saves an intermediate object creation
-                               traces.filter_map { |t| encode_one(t) }
-                             else
-                               traces.map { |t| encode_one(t) }.reject(&:nil?)
-                             end
+            encoded_traces = Core::Utils::EnumerableCompat.filter_map(traces) do |trace|
+              encode_one(trace)
+            end
 
             Datadog::Core::Chunker.chunk_by_size(encoded_traces, max_size).map do |chunk|
               [encoder.join(chunk), chunk.size]
@@ -77,11 +75,16 @@ module Datadog
           private
 
           def encode_one(trace)
-            encoded = Encoder.encode_trace(encoder, trace)
+            encoded = Encoder.encode_trace(
+              encoder,
+              trace,
+              logger: logger,
+              native_events_supported: @native_events_supported
+            )
 
             if encoded.size > max_size
               # This single trace is too large, we can't flush it
-              Datadog.logger.debug { "Dropping trace. Payload too large: '#{trace.inspect}'" }
+              logger.debug { "Dropping trace. Payload too large: '#{trace.inspect}'" }
               Datadog.health_metrics.transport_trace_too_large(1)
 
               return nil
@@ -95,17 +98,18 @@ module Datadog
         module Encoder
           module_function
 
-          def encode_trace(encoder, trace)
+          def encode_trace(encoder, trace, logger:, native_events_supported:)
             # Format the trace for transport
-            TraceFormatter.format!(trace)
+            TraceFormatter.format!(trace, transport: "ruby")
 
             # Make the trace serializable
-            serializable_trace = SerializableTrace.new(trace)
-
-            Datadog.logger.debug { "Flushing trace: #{JSON.dump(serializable_trace)}" }
+            serializable_trace = SerializableTrace.new(trace, native_events_supported: native_events_supported)
 
             # Encode the trace
-            encoder.encode(serializable_trace)
+            encoder.encode(serializable_trace).tap do |encoded|
+              # Print the actual serialized trace, since the encoder can change make non-trivial changes
+              logger.debug { "Flushing trace: #{encoder.decode(encoded)}" }
+            end
           end
         end
 
@@ -114,24 +118,22 @@ module Datadog
         # This class initializes the HTTP client, breaks down large
         # batches of traces into smaller chunks and handles
         # API version downgrade handshake.
-        class Transport
-          attr_reader :client, :apis, :default_api, :current_api_id
-
-          def initialize(apis, default_api)
-            @apis = apis
-            @default_api = default_api
-
-            change_api!(default_api)
-          end
+        class Transport < Core::Transport::Transport
+          include SpanEventsNegotiation
+          self.http_client_class = Tracing::Transport::HTTP::Client
 
           def send_traces(traces)
             encoder = current_api.encoder
-            chunker = Datadog::Tracing::Transport::Traces::Chunker.new(encoder)
+            chunker = Datadog::Tracing::Transport::Traces::Chunker.new(
+              encoder,
+              logger: logger,
+              native_events_supported: native_events_supported?
+            )
 
             responses = chunker.encode_in_chunks(traces.lazy).map do |encoded_traces, trace_count|
-              request = Request.new(EncodedParcel.new(encoded_traces, trace_count))
+              request = Request.new(Parcel.new(encoded_traces, trace_count: trace_count))
 
-              client.send_traces_payload(request).tap do |response|
+              client.send_request(:traces, request).tap do |response|
                 if downgrade?(response)
                   downgrade!
                   return send_traces(traces)
@@ -159,63 +161,7 @@ module Datadog
           end
 
           def stats
-            @client.stats
-          end
-
-          def current_api
-            apis[@current_api_id]
-          end
-
-          private
-
-          def downgrade?(response)
-            return false unless apis.fallbacks.key?(@current_api_id)
-
-            response.not_found? || response.unsupported?
-          end
-
-          def downgrade!
-            downgrade_api_id = apis.fallbacks[@current_api_id]
-            raise NoDowngradeAvailableError, @current_api_id if downgrade_api_id.nil?
-
-            change_api!(downgrade_api_id)
-          end
-
-          def change_api!(api_id)
-            raise UnknownApiVersionError, api_id unless apis.key?(api_id)
-
-            @current_api_id = api_id
-            @client = HTTP::Client.new(current_api)
-          end
-
-          # Raised when configured with an unknown API version
-          class UnknownApiVersionError < StandardError
-            attr_reader :version
-
-            def initialize(version)
-              super
-
-              @version = version
-            end
-
-            def message
-              "No matching transport API for version #{version}!"
-            end
-          end
-
-          # Raised when configured with an unknown API version
-          class NoDowngradeAvailableError < StandardError
-            attr_reader :version
-
-            def initialize(version)
-              super
-
-              @version = version
-            end
-
-            def message
-              "No downgrade from transport API version #{version} is available!"
-            end
+            client.stats
           end
         end
       end

data/lib/datadog/tracing/utils.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require_relative '../core/utils/forking'
-require_relative '../core/utils/time'
+require_relative "../core/utils/forking"
+require_relative "../core/utils/time"
 
 module Datadog
   module Tracing
@@ -24,7 +24,7 @@ module Datadog
 
       # While we only generate 63-bit integers due to limitations in other languages, we support
       # parsing 64-bit integers for distributed tracing since an upstream system may generate one
-      EXTERNAL_MAX_ID = 1 << 64
+      EXTERNAL_MAX_ID = (1 << 64) - 1
 
       # We use a custom random number generator because we want no interference
       # with the default one. Using the default prng, we could break code that

data/lib/datadog/tracing/workers.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require_relative 'buffer'
-require_relative 'pipeline'
+require_relative "buffer"
+require_relative "pipeline"
 
 module Datadog
   module Tracing
@@ -18,8 +18,7 @@ module Datadog
         BACK_OFF_MAX = 5
         DEFAULT_SHUTDOWN_TIMEOUT = 1
 
-        attr_reader \
-          :trace_buffer
+        attr_reader :trace_buffer, :logger
 
         def initialize(options = {})
           @transport = options[:transport]
@@ -42,6 +41,8 @@ module Datadog
           @mutex = Mutex.new
           @worker = nil
           @run = false
+
+          @logger = options.fetch(:logger)
         end
 
         # Callback function that process traces and executes the +send_traces()+ method.
@@ -52,12 +53,13 @@ module Datadog
             traces = @trace_buffer.pop
             traces = Pipeline.process!(traces)
             @trace_task.call(traces, @transport) unless @trace_task.nil? || traces.empty?
-          rescue StandardError => e
+          rescue => e
             # ensures that the thread will not die because of an exception.
             # TODO[manu]: findout the reason and reschedule the send if it's not
             # a fatal exception
-            Datadog.logger.warn(
-              "Error during traces flush: dropped #{traces.length} items. Cause: #{e} Location: #{Array(e.backtrace).first}"
+            logger.warn(
+              "Error during traces flush: dropped #{traces.length} items. " \
+              "Cause: #{e.class}: #{e.message} Location: #{Array(e.backtrace).first}"
             )
           end
         end
@@ -68,9 +70,9 @@ module Datadog
             return if @run
 
             @run = true
-            Datadog.logger.debug { "Starting thread for: #{self}" }
+            logger.debug { "Starting thread for: #{self}" }
             @worker = Thread.new { perform }
-            @worker.name = self.class.name unless Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.3')
+            @worker.name = self.class.name
             @worker.thread_variable_set(:fork_safe, true)
 
             nil
@@ -106,7 +108,7 @@ module Datadog
 
         private
 
-        alias flush_data callback_traces
+        alias_method :flush_data, :callback_traces
 
         def perform
           loop do

data/lib/datadog/tracing/writer.rb

@@ -1,33 +1,36 @@
 # frozen_string_literal: true
 
-require_relative 'event'
-require_relative 'runtime/metrics'
-require_relative 'workers'
+require_relative "event"
+require_relative "runtime/metrics"
+require_relative "workers"
 
-require_relative 'transport/http'
+require_relative "transport/http"
 
 module Datadog
   module Tracing
     # Processor that sends traces and metadata to the agent
-    # DEV: Our goal is for {Datadog::Tracing::Workers::TraceWriter} to replace this class in the future
     # @public_api
     class Writer
       attr_reader \
+        :logger,
         :transport,
         :worker,
-        :events
+        :events,
+        :agent_settings
 
+      # DEV-3.0: change to keyword arguments
       def initialize(options = {})
+        @logger = options[:logger] || Datadog.logger
+
         # writer and transport parameters
         @buff_size = options.fetch(:buffer_size, Workers::AsyncTransport::DEFAULT_BUFFER_MAX_SIZE)
         @flush_interval = options.fetch(:flush_interval, Workers::AsyncTransport::DEFAULT_FLUSH_INTERVAL)
         transport_options = options.fetch(:transport_options, {})
-
-        transport_options[:agent_settings] = options[:agent_settings] if options.key?(:agent_settings)
+        @agent_settings = options[:agent_settings]
 
         # transport and buffers
         @transport = options.fetch(:transport) do
-          Transport::HTTP.default(**transport_options)
+          Transport::HTTP.default(agent_settings: agent_settings, logger: logger, **transport_options)
         end
 
         @shutdown_timeout = options.fetch(:shutdown_timeout, Workers::AsyncTransport::DEFAULT_SHUTDOWN_TIMEOUT)
@@ -68,21 +71,6 @@ module Datadog
         end
       end
 
-      # spawns a worker for spans; they share the same transport which is thread-safe
-      # @!visibility private
-      def start_worker
-        @trace_handler = ->(items, transport) { send_spans(items, transport) }
-        @worker = Workers::AsyncTransport.new(
-          transport: @transport,
-          buffer_size: @buff_size,
-          on_trace: @trace_handler,
-          interval: @flush_interval,
-          shutdown_timeout: @shutdown_timeout
-        )
-
-        @worker.start
-      end
-
       # Gracefully shuts down this writer.
       #
       # Once stopped methods calls won't fail, but
@@ -93,19 +81,6 @@ module Datadog
         @mutex_after_fork.synchronize { stop_worker }
       end
 
-      def stop_worker
-        @stopped = true
-
-        return if @worker.nil?
-
-        @worker.stop
-        @worker = nil
-
-        true
-      end
-
-      private :start_worker, :stop_worker
-
       # flush spans to the trace-agent, handles spans only
       # @!visibility private
       def send_spans(traces, transport)
@@ -147,7 +122,7 @@ module Datadog
         if worker_local
           worker_local.enqueue_trace(trace)
         elsif !@stopped
-          Datadog.logger.debug('Writer either failed to start or was stopped before #write could complete')
+          logger.debug("Writer either failed to start or was stopped before #write could complete")
         end
       end
 
@@ -155,7 +130,7 @@ module Datadog
       def stats
         {
           traces_flushed: @traces_flushed,
-          transport: @transport.stats
+          transport: @transport.stats,
         }
       end
 
@@ -179,6 +154,46 @@ module Datadog
 
       private
 
+      # spawns a worker for spans; they share the same transport which is thread-safe
+      # @!visibility private
+      def start_worker
+        @trace_handler = ->(items, transport) { send_spans(items, transport) }
+        @worker = Workers::AsyncTransport.new(
+          transport: @transport,
+          buffer_size: @buff_size,
+          on_trace: @trace_handler,
+          interval: @flush_interval,
+          shutdown_timeout: @shutdown_timeout,
+          logger: logger,
+        )
+
+        @worker.start
+      end
+
+      def stop_worker
+        @stopped = true
+
+        if @worker
+          @worker.stop
+          @worker = nil
+        end
+
+        # Stopping a writer is permanent: once `@stopped` is set, `#start`
+        # refuses to spin up a new worker, so this writer (and the transport it
+        # owns) is being discarded for good. Forks do not stop the writer
+        # (`#start` re-creates the worker for the new pid while reusing the same
+        # transport), so this only runs on real teardown/reconfiguration.
+        #
+        # Deterministically release transports that hold native resources
+        # (e.g. the native trace exporter's Rust runtime and process-global
+        # fork hooks) instead of waiting on the GC finalizer. The default HTTP
+        # transport has no `#close` and is left untouched. `#close` is
+        # idempotent, so repeated `#stop` calls are safe.
+        @transport.close if @transport.respond_to?(:close)
+
+        true
+      end
+
       def reset_stats!
         @traces_flushed = 0
         @transport.stats.reset!

data/lib/datadog/tracing.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require_relative 'core'
-require_relative 'tracing/pipeline'
+require_relative "core"
+require_relative "tracing/pipeline"
 
 module Datadog
   # Datadog APM tracing public API.
@@ -26,7 +26,6 @@ module Datadog
         id: nil,
         &block
       )
-
         tracer.trace(
           name,
           continue_from: continue_from,
@@ -82,7 +81,7 @@ module Datadog
       # @public_api
       def keep!
         trace = active_trace
-        active_trace.keep! if trace
+        trace&.keep!
       end
 
       # (see Datadog::Tracing::TraceSegment#reject!)
@@ -90,7 +89,7 @@ module Datadog
       # @public_api
       def reject!
         trace = active_trace
-        active_trace.reject! if trace
+        trace&.reject!
       end
 
       # (see Datadog::Tracing::Tracer#active_correlation)
@@ -122,11 +121,25 @@ module Datadog
       # @return [String] correlation information; or an empty String if Tracing is disabled (`!enabled?`)
       # @public_api
       def log_correlation
-        return '' unless enabled?
+        return "" unless enabled?
 
         correlation.to_log_format
       end
 
+      # Returns the baggage for the current trace.
+      #
+      # If there is no active trace, a new one is created.
+      #
+      # @return [Datadog::Tracing::Distributed::Baggage] The baggage for the current trace.
+      # @public_api
+      def baggage
+        # Baggage should not be dependent on there being an active trace.
+        # So we create a new TraceOperation if there isn't one.
+        active_trace = self.active_trace || tracer.continue_trace!(nil)
+        active_trace.baggage ||= {}
+        active_trace.baggage
+      end
+
       # Gracefully shuts down the tracer.
       #
       # The public tracing API will still respond to method calls as usual

data/lib/datadog/version.rb

@@ -3,15 +3,15 @@
 module Datadog
   module VERSION
     MAJOR = 2
-    MINOR = 4
+    MINOR = 44
     PATCH = 0
     PRE = nil
     BUILD = nil
     # PRE and BUILD above are modified for dev gems during gem build GHA workflow
 
-    STRING = [MAJOR, MINOR, PATCH, PRE, BUILD].compact.join('.')
+    STRING = [MAJOR, MINOR, PATCH, PRE, BUILD].compact.join(".")
 
-    MINIMUM_RUBY_VERSION = '2.5.0'
+    MINIMUM_RUBY_VERSION = "2.5.0"
 
     # Restrict the installation of this gem with untested future versions of Ruby.
     #
@@ -21,6 +21,6 @@ module Datadog
     # To allow testing with the next unreleased version of Ruby, the version check is performed
     # as `< #{MAXIMUM_RUBY_VERSION}`, meaning prereleases of MAXIMUM_RUBY_VERSION are allowed
     # but not stable MAXIMUM_RUBY_VERSION releases.
-    MAXIMUM_RUBY_VERSION = '3.5'
+    MAXIMUM_RUBY_VERSION = "4.1"
   end
 end