datadog 2.10.0 → 2.12.2

data/lib/datadog/tracing/metadata/metastruct.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+
+module Datadog
+  module Tracing
+    module Metadata
+      # This class is a data structure that is used to store
+      # complex metadata, such as an array of objects.
+      #
+      # It is serialized to MessagePack format when sent to the agent.
+      class Metastruct
+        extend Forwardable
+
+        def_delegators :@metastruct, :[], :[]=, :to_h
+
+        def initialize
+          @metastruct = {}
+        end
+
+        def to_msgpack(packer = nil)
+          # JRuby doesn't pass the packer
+          packer ||= MessagePack::Packer.new
+
+          packer.write(@metastruct.transform_values(&:to_msgpack))
+        end
+
+        def pretty_print(q)
+          q.seplist @metastruct.each do |key, value|
+            q.text "#{key} => #{value}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/metadata/metastruct_tagging.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require_relative 'metastruct'
+
+module Datadog
+  module Tracing
+    module Metadata
+      # Adds data storage for the `meta_struct` field.
+      #
+      # This field is used to send more complex data like an array of objects
+      # in MessagePack format to the agent, and has no size limitations.
+      #
+      # The agent fully supports meta_struct from version v7.35.0 (April 2022).
+      #
+      # On versions older than v7.35.0, sending traces containing meta_struct
+      # has no unexpected side-effects; traces are sent to the backend as expected,
+      # while the meta_struct field is stripped.
+      module MetastructTagging
+        # Set the given key / value tag pair on the metastruct.
+        #
+        # A valid example is:
+        #
+        #   span.set_metastruct_tag('_dd.stack', [])
+        def set_metastruct_tag(key, value)
+          metastruct[key] = value
+        end
+
+        # Return the metastruct tag value for the given key,
+        # returns nil if the key doesn't exist.
+        def get_metastruct_tag(key)
+          metastruct[key]
+        end
+
+        private
+
+        def metastruct
+          @metastruct ||= Metastruct.new
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/metadata.rb

@@ -2,6 +2,7 @@
 
 require_relative 'metadata/analytics'
 require_relative 'metadata/tagging'
+require_relative 'metadata/metastruct_tagging'
 require_relative 'metadata/errors'
 
 module Datadog
@@ -10,6 +11,7 @@ module Datadog
     module Metadata
       def self.included(base)
         base.include(Metadata::Tagging)
+        base.include(Metadata::MetastructTagging)
         base.include(Metadata::Errors)
 
         # Additional extensions

data/lib/datadog/tracing/span.rb

@@ -33,6 +33,9 @@ module Datadog
         :status,
         :trace_id
 
+      attr_reader \
+        :metastruct
+
       attr_writer \
         :duration
 
@@ -54,6 +57,7 @@ module Datadog
         id: nil,
         meta: nil,
         metrics: nil,
+        metastruct: nil,
         parent_id: 0,
         resource: name,
         service: nil,
@@ -76,6 +80,7 @@ module Datadog
 
         @meta = meta || {}
         @metrics = metrics || {}
+        @metastruct = metastruct || {}
         @status = status || 0
 
         # start_time and end_time track wall clock. In Ruby, wall clock
@@ -144,6 +149,7 @@ module Datadog
           error: @status,
           meta: @meta,
           metrics: @metrics,
+          meta_struct: @metastruct.to_h,
           name: @name,
           parent_id: @parent_id,
           resource: @resource,
@@ -185,12 +191,15 @@ module Datadog
               q.text "#{key} => #{value}"
             end
           end
-          q.group(2, 'Metrics: [', ']') do
+          q.group(2, 'Metrics: [', "]\n") do
             q.breakable
             q.seplist @metrics.each do |key, value|
               q.text "#{key} => #{value}"
             end
           end
+          q.group(2, 'Metastruct: [', ']') do
+            metastruct.pretty_print(q)
+          end
         end
       end
 

data/lib/datadog/tracing/span_operation.rb

@@ -289,6 +289,7 @@ module Datadog
           id: @id,
           meta: meta,
           metrics: metrics,
+          metastruct: metastruct,
           name: @name,
           parent_id: @parent_id,
           resource: @resource,
@@ -328,12 +329,15 @@ module Datadog
               q.text "#{key} => #{value}"
             end
           end
-          q.group(2, 'Metrics: [', ']') do
+          q.group(2, 'Metrics: [', "]\n") do
             q.breakable
             q.seplist metrics.each do |key, value|
               q.text "#{key} => #{value}"
             end
           end
+          q.group(2, 'Metastruct: [', ']') do
+            metastruct.pretty_print(q)
+          end
         end
       end
 
@@ -456,6 +460,7 @@ module Datadog
           id: @id,
           meta: Core::Utils::SafeDup.frozen_or_dup(meta),
           metrics: Core::Utils::SafeDup.frozen_or_dup(metrics),
+          metastruct: Core::Utils::SafeDup.frozen_or_dup(metastruct),
           parent_id: @parent_id,
           resource: @resource,
           service: @service,

data/lib/datadog/tracing/sync_writer.rb

@@ -17,17 +17,22 @@ module Datadog
     # @public_api
     class SyncWriter
       attr_reader \
+        :logger,
         :events,
-        :transport
+        :transport,
+        :agent_settings
 
       # @param [Datadog::Tracing::Transport::Traces::Transport] transport a custom transport instance.
       #   If provided, overrides `transport_options` and `agent_settings`.
       # @param [Hash<Symbol,Object>] transport_options options for the default transport instance.
       # @param [Datadog::Tracing::Configuration::AgentSettingsResolver::AgentSettings] agent_settings agent options for
       #   the default transport instance.
-      def initialize(transport: nil, transport_options: {}, agent_settings: nil)
+      def initialize(transport: nil, transport_options: {}, agent_settings: nil, logger: Datadog.logger)
+        @logger = logger
+        @agent_settings = agent_settings
+
         @transport = transport || begin
-          transport_options[:agent_settings] = agent_settings if agent_settings
+          transport_options = transport_options.merge(agent_settings: agent_settings) if agent_settings
           Transport::HTTP.default(**transport_options)
         end
 
@@ -40,7 +45,7 @@ module Datadog
       def write(trace)
         flush_trace(trace)
       rescue => e
-        Datadog.logger.debug(e)
+        logger.debug(e)
       end
 
       # Does nothing.

data/lib/datadog/tracing/tracer.rb

@@ -28,7 +28,8 @@ module Datadog
         :provider,
         :sampler,
         :span_sampler,
-        :tags
+        :tags,
+        :logger
 
       attr_accessor \
         :default_service,
@@ -48,21 +49,28 @@ module Datadog
       # @param tags [Hash] default tags added to all spans
       # @param writer [Datadog::Tracing::Writer] consumes traces returned by the provided +trace_flush+
       def initialize(
+        # rubocop:disable Style/KeywordParametersOrder
+        # https://github.com/rubocop/rubocop/issues/13933
         trace_flush: Flush::Finished.new,
         context_provider: DefaultContextProvider.new,
         default_service: Core::Environment::Ext::FALLBACK_SERVICE_NAME,
         enabled: true,
+        logger: Datadog.logger,
         sampler: Sampling::PrioritySampler.new(
           base_sampler: Sampling::AllSampler.new,
           post_sampler: Sampling::RuleSampler.new
         ),
         span_sampler: Sampling::Span::Sampler.new,
         tags: {},
-        writer: Writer.new
+        # writer is not defaulted because creating it requires agent_settings,
+        # which we do not have here and otherwise do not need.
+        writer:
+        # rubocop:enable Style/KeywordParametersOrder
       )
         @trace_flush = trace_flush
         @default_service = default_service
         @enabled = enabled
+        @logger = logger
         @provider = context_provider
         @sampler = sampler
         @span_sampler = span_sampler
@@ -146,7 +154,7 @@ module Datadog
                     active_trace
                   end
         rescue StandardError => e
-          Datadog.logger.debug { "Failed to trace: #{e}" }
+          logger.debug { "Failed to trace: #{e}" }
 
           # Tracing failed: fallback and run code without tracing.
           return skip_trace(name, &block)
@@ -268,7 +276,7 @@ module Datadog
           @sampler.sample!(trace_op)
         rescue StandardError => e
           SAMPLE_TRACE_LOG_ONLY_ONCE.run do
-            Datadog.logger.warn { "Failed to sample trace: #{e.class.name} #{e} at #{Array(e.backtrace).first}" }
+            logger.warn { "Failed to sample trace: #{e.class.name} #{e} at #{Array(e.backtrace).first}" }
           end
         end
       end
@@ -488,7 +496,7 @@ module Datadog
           @span_sampler.sample!(trace_op, span)
         rescue StandardError => e
           SAMPLE_SPAN_LOG_ONLY_ONCE.run do
-            Datadog.logger.warn { "Failed to sample span: #{e.class.name} #{e} at #{Array(e.backtrace).first}" }
+            logger.warn { "Failed to sample span: #{e.class.name} #{e} at #{Array(e.backtrace).first}" }
           end
         end
       end
@@ -504,7 +512,7 @@ module Datadog
           write(trace) if trace && !trace.empty?
         rescue StandardError => e
           FLUSH_TRACE_LOG_ONLY_ONCE.run do
-            Datadog.logger.warn { "Failed to flush trace: #{e.class.name} #{e} at #{Array(e.backtrace).first}" }
+            logger.warn { "Failed to flush trace: #{e.class.name} #{e} at #{Array(e.backtrace).first}" }
           end
         end
       end
@@ -518,7 +526,7 @@ module Datadog
         return unless trace && @writer
 
         if Datadog.configuration.diagnostics.debug
-          Datadog.logger.debug { "Writing #{trace.length} spans (enabled: #{@enabled})\n#{trace.spans.pretty_inspect}" }
+          logger.debug { "Writing #{trace.length} spans (enabled: #{@enabled})\n#{trace.spans.pretty_inspect}" }
         end
 
         @writer.write(trace)

data/lib/datadog/tracing/transport/http/api.rb

@@ -3,7 +3,8 @@
 require_relative '../../../core/encoding'
 
 require_relative '../../../core/transport/http/api/map'
-require_relative 'api/spec'
+require_relative '../../../core/transport/http/api/instance'
+require_relative '../../../core/transport/http/api/spec'
 
 require_relative 'traces'
 
@@ -20,7 +21,7 @@ module Datadog
           module_function
 
           def defaults
-            Datadog::Core::Transport::HTTP::API::Map[
+            Core::Transport::HTTP::API::Map[
               V4 => Spec.new do |s|
                 s.traces = Traces::API::Endpoint.new(
                   '/v0.4/traces',
@@ -36,6 +37,14 @@ module Datadog
               end,
             ].with_fallbacks(V4 => V3)
           end
+
+          class Instance < Core::Transport::HTTP::API::Instance
+            include Traces::API::Instance
+          end
+
+          class Spec < Core::Transport::HTTP::API::Spec
+            include Traces::API::Spec
+          end
         end
       end
     end

data/lib/datadog/tracing/transport/http/traces.rb

@@ -6,7 +6,6 @@ require_relative '../traces'
 require_relative 'client'
 require_relative '../../../core/transport/http/response'
 require_relative '../../../core/transport/http/api/endpoint'
-require_relative 'api/instance'
 
 module Datadog
   module Tracing
@@ -143,8 +142,6 @@ module Datadog
 
           # Add traces behavior to transport components
           HTTP::Client.include(Traces::Client)
-          HTTP::API::Spec.include(Traces::API::Spec)
-          HTTP::API::Instance.include(Traces::API::Instance)
         end
       end
     end

data/lib/datadog/tracing/transport/http.rb

@@ -1,15 +1,10 @@
 # frozen_string_literal: true
 
-require 'uri'
-
 require_relative '../../core/environment/container'
 require_relative '../../core/environment/ext'
 require_relative '../../core/transport/ext'
-require_relative '../../core/transport/http/adapters/net'
-require_relative '../../core/transport/http/adapters/test'
-require_relative '../../core/transport/http/adapters/unix_socket'
+require_relative '../../core/transport/http'
 require_relative 'http/api'
-require_relative 'http/builder'
 require_relative '../../../datadog/version'
 
 module Datadog
@@ -17,30 +12,22 @@ module Datadog
     module Transport
       # Namespace for HTTP transport components
       module HTTP
-        # NOTE: Due to... legacy reasons... This class likes having a default `AgentSettings` instance to fall back to.
-        # Because we generate this instance with an empty instance of `Settings`, the resulting `AgentSettings` below
-        # represents only settings specified via environment variables + the usual defaults.
-        #
-        # DO NOT USE THIS IN NEW CODE, as it ignores any settings specified by users via `Datadog.configure`.
-        DO_NOT_USE_ENVIRONMENT_AGENT_SETTINGS = Datadog::Core::Configuration::AgentSettingsResolver.call(
-          Datadog::Core::Configuration::Settings.new,
-          logger: nil,
-        )
-
         module_function
 
         # Builds a new Transport::HTTP::Client
-        def new(&block)
-          Builder.new(&block).to_transport
+        def new(klass, &block)
+          Core::Transport::HTTP.build(
+            api_instance_class: API::Instance, &block
+          ).to_transport(klass)
         end
 
         # Builds a new Transport::HTTP::Client with default settings
         # Pass a block to override any settings.
         def default(
-          agent_settings: DO_NOT_USE_ENVIRONMENT_AGENT_SETTINGS,
+          agent_settings:,
           **options
         )
-          new do |transport|
+          new(Transport::Traces::Transport) do |transport|
             transport.adapter(agent_settings)
             transport.headers default_headers
 
@@ -84,17 +71,6 @@ module Datadog
         def default_adapter
           Datadog::Core::Configuration::Ext::Agent::HTTP::ADAPTER
         end
-
-        # Add adapters to registry
-        Builder::REGISTRY.set(
-          Datadog::Core::Transport::HTTP::Adapters::Net,
-          Datadog::Core::Configuration::Ext::Agent::HTTP::ADAPTER
-        )
-        Builder::REGISTRY.set(Datadog::Core::Transport::HTTP::Adapters::Test, Datadog::Core::Transport::Ext::Test::ADAPTER)
-        Builder::REGISTRY.set(
-          Datadog::Core::Transport::HTTP::Adapters::UnixSocket,
-          Datadog::Core::Transport::Ext::UnixSocket::ADAPTER
-        )
       end
     end
   end

data/lib/datadog/tracing/transport/serializable_trace.rb

@@ -14,7 +14,7 @@ module Datadog
 
         # @param trace [Datadog::Trace] the trace to serialize
         # @param native_events_supported [Boolean] whether the agent supports span events as a top-level field
-        def initialize(trace, native_events_supported = false)
+        def initialize(trace, native_events_supported:)
           @trace = trace
           @native_events_supported = native_events_supported
         end
@@ -29,13 +29,17 @@ module Datadog
         # @param packer [MessagePack::Packer] serialization buffer, can be +nil+ with JRuby
         def to_msgpack(packer = nil)
           # As of 1.3.3, JRuby implementation doesn't pass an existing packer
-          trace.spans.map { |s| SerializableSpan.new(s, @native_events_supported) }.to_msgpack(packer)
+          trace.spans.map do |s|
+            SerializableSpan.new(s, native_events_supported: @native_events_supported)
+          end.to_msgpack(packer)
         end
 
         # JSON serializer interface.
         # Used by older version of the transport.
         def to_json(*args)
-          trace.spans.map { |s| SerializableSpan.new(s, @native_events_supported).to_hash }.to_json(*args)
+          trace.spans.map do |s|
+            SerializableSpan.new(s, native_events_supported: @native_events_supported).to_hash
+          end.to_json(*args)
         end
       end
 
@@ -46,7 +50,7 @@ module Datadog
 
         # @param span [Datadog::Span] the span to serialize
         # @param native_events_supported [Boolean] whether the agent supports span events as a top-level field
-        def initialize(span, native_events_supported)
+        def initialize(span, native_events_supported:)
           @span = span
           @trace_id = Tracing::Utils::TraceId.to_low_order(span.trace_id)
           @native_events_supported = native_events_supported
@@ -65,7 +69,7 @@ module Datadog
         def to_msgpack(packer = nil)
           packer ||= MessagePack::Packer.new
 
-          number_of_elements_to_write = 11
+          number_of_elements_to_write = 12
 
           number_of_elements_to_write += 1 if span.events.any? && @native_events_supported
 
@@ -113,6 +117,8 @@ module Datadog
           packer.write(span.meta)
           packer.write('metrics')
           packer.write(span.metrics)
+          packer.write('meta_struct')
+          packer.write(span.metastruct)
           packer.write('span_links')
           packer.write(span.links.map(&:to_hash))
           packer.write('error')

data/lib/datadog/tracing/transport/traces.rb

@@ -50,8 +50,9 @@ module Datadog
           #
           # @param encoder [Datadog::Core::Encoding::Encoder]
           # @param max_size [String] maximum acceptable payload size
-          def initialize(encoder, max_size: DEFAULT_MAX_PAYLOAD_SIZE)
+          def initialize(encoder, native_events_supported:, max_size: DEFAULT_MAX_PAYLOAD_SIZE)
             @encoder = encoder
+            @native_events_supported = native_events_supported
             @max_size = max_size
           end
 
@@ -77,7 +78,7 @@ module Datadog
           private
 
           def encode_one(trace)
-            encoded = Encoder.encode_trace(encoder, trace)
+            encoded = Encoder.encode_trace(encoder, trace, native_events_supported: @native_events_supported)
 
             if encoded.size > max_size
               # This single trace is too large, we can't flush it
@@ -95,17 +96,18 @@ module Datadog
         module Encoder
           module_function
 
-          def encode_trace(encoder, trace)
+          def encode_trace(encoder, trace, native_events_supported:)
             # Format the trace for transport
             TraceFormatter.format!(trace)
 
             # Make the trace serializable
-            serializable_trace = SerializableTrace.new(trace)
-
-            Datadog.logger.debug { "Flushing trace: #{JSON.dump(serializable_trace)}" }
+            serializable_trace = SerializableTrace.new(trace, native_events_supported: native_events_supported)
 
             # Encode the trace
-            encoder.encode(serializable_trace)
+            encoder.encode(serializable_trace).tap do |encoded|
+              # Print the actual serialized trace, since the encoder can change make non-trivial changes
+              Datadog.logger.debug { "Flushing trace: #{encoder.decode(encoded)}" }
+            end
           end
         end
 
@@ -126,7 +128,10 @@ module Datadog
 
           def send_traces(traces)
             encoder = current_api.encoder
-            chunker = Datadog::Tracing::Transport::Traces::Chunker.new(encoder)
+            chunker = Datadog::Tracing::Transport::Traces::Chunker.new(
+              encoder,
+              native_events_supported: native_events_supported?
+            )
 
             responses = chunker.encode_in_chunks(traces.lazy).map do |encoded_traces, trace_count|
               request = Request.new(EncodedParcel.new(encoded_traces, trace_count))
@@ -188,6 +193,18 @@ module Datadog
             @client = HTTP::Client.new(current_api)
           end
 
+          # Queries the agent for native span events serialization support.
+          # This changes how the serialization of span events performed.
+          def native_events_supported?
+            return @native_events_supported if defined?(@native_events_supported)
+
+            if (res = Datadog.send(:components).agent_info.fetch)
+              @native_events_supported = res.span_events == true
+            else
+              false
+            end
+          end
+
           # Raised when configured with an unknown API version
           class UnknownApiVersionError < StandardError
             attr_reader :version

data/lib/datadog/tracing/workers/trace_writer.rb

@@ -17,13 +17,20 @@ module Datadog
       # Writes traces to transport synchronously
       class TraceWriter < Core::Worker
         attr_reader \
-          :transport
+          :logger,
+          :transport,
+          :agent_settings
 
         # rubocop:disable Lint/MissingSuper
         def initialize(options = {})
+          @logger = options[:logger] || Datadog.logger
+
           transport_options = options.fetch(:transport_options, {})
 
-          transport_options[:agent_settings] = options[:agent_settings] if options.key?(:agent_settings)
+          if options.key?(:agent_settings)
+            @agent_settings = options[:agent_settings]
+            transport_options = transport_options.merge(agent_settings: @agent_settings)
+          end
 
           @transport = options.fetch(:transport) do
             Datadog::Tracing::Transport::HTTP.default(**transport_options)
@@ -43,7 +50,7 @@ module Datadog
           traces = process_traces(traces)
           flush_traces(traces)
         rescue StandardError => e
-          Datadog.logger.warn(
+          logger.warn(
             "Error while writing traces: dropped #{traces.length} items. Cause: #{e} Location: #{Array(e.backtrace).first}"
           )
         end

data/lib/datadog/tracing/workers.rb

@@ -18,8 +18,7 @@ module Datadog
         BACK_OFF_MAX = 5
         DEFAULT_SHUTDOWN_TIMEOUT = 1
 
-        attr_reader \
-          :trace_buffer
+        attr_reader :trace_buffer, :logger
 
         def initialize(options = {})
           @transport = options[:transport]
@@ -42,6 +41,8 @@ module Datadog
           @mutex = Mutex.new
           @worker = nil
           @run = false
+
+          @logger = options.fetch(:logger)
         end
 
         # Callback function that process traces and executes the +send_traces()+ method.
@@ -56,7 +57,7 @@ module Datadog
             # ensures that the thread will not die because of an exception.
             # TODO[manu]: findout the reason and reschedule the send if it's not
             # a fatal exception
-            Datadog.logger.warn(
+            logger.warn(
               "Error during traces flush: dropped #{traces.length} items. Cause: #{e} Location: #{Array(e.backtrace).first}"
             )
           end
@@ -68,7 +69,7 @@ module Datadog
             return if @run
 
             @run = true
-            Datadog.logger.debug { "Starting thread for: #{self}" }
+            logger.debug { "Starting thread for: #{self}" }
             @worker = Thread.new { perform }
             @worker.name = self.class.name
             @worker.thread_variable_set(:fork_safe, true)

data/lib/datadog/tracing/writer.rb

@@ -13,17 +13,24 @@ module Datadog
     # @public_api
     class Writer
       attr_reader \
+        :logger,
         :transport,
         :worker,
-        :events
+        :events,
+        :agent_settings
 
       def initialize(options = {})
+        @logger = options[:logger] || Datadog.logger
+
         # writer and transport parameters
         @buff_size = options.fetch(:buffer_size, Workers::AsyncTransport::DEFAULT_BUFFER_MAX_SIZE)
         @flush_interval = options.fetch(:flush_interval, Workers::AsyncTransport::DEFAULT_FLUSH_INTERVAL)
         transport_options = options.fetch(:transport_options, {})
 
-        transport_options[:agent_settings] = options[:agent_settings] if options.key?(:agent_settings)
+        if options.key?(:agent_settings)
+          @agent_settings = options[:agent_settings]
+          transport_options = transport_options.merge(agent_settings: @agent_settings)
+        end
 
         # transport and buffers
         @transport = options.fetch(:transport) do
@@ -119,7 +126,7 @@ module Datadog
         if worker_local
           worker_local.enqueue_trace(trace)
         elsif !@stopped
-          Datadog.logger.debug('Writer either failed to start or was stopped before #write could complete')
+          logger.debug('Writer either failed to start or was stopped before #write could complete')
         end
       end
 
@@ -160,7 +167,8 @@ module Datadog
           buffer_size: @buff_size,
           on_trace: @trace_handler,
           interval: @flush_interval,
-          shutdown_timeout: @shutdown_timeout
+          shutdown_timeout: @shutdown_timeout,
+          logger: logger,
         )
 
         @worker.start

data/lib/datadog/version.rb

@@ -3,8 +3,8 @@
 module Datadog
   module VERSION
     MAJOR = 2
-    MINOR = 10
-    PATCH = 0
+    MINOR = 12
+    PATCH = 2
     PRE = nil
     BUILD = nil
     # PRE and BUILD above are modified for dev gems during gem build GHA workflow