ddtrace 1.4.2 → 1.19.0

data/lib/datadog/tracing/transport/traces.rb

@@ -0,0 +1,224 @@
+# frozen_string_literal: true
+
+require_relative '../../core/chunker'
+require_relative '../../core/transport/parcel'
+require_relative '../../core/transport/request'
+require_relative 'serializable_trace'
+require_relative 'trace_formatter'
+
+module Datadog
+  module Tracing
+    module Transport
+      module Traces
+        # Data transfer object for encoded traces
+        class EncodedParcel
+          include Datadog::Core::Transport::Parcel
+
+          attr_reader :trace_count
+
+          def initialize(data, trace_count)
+            super(data)
+            @trace_count = trace_count
+          end
+
+          def count
+            data.length
+          end
+        end
+
+        # Traces request
+        class Request < Datadog::Core::Transport::Request
+        end
+
+        # Traces response
+        module Response
+          attr_reader :service_rates, :trace_count
+        end
+
+        # Traces chunker
+        class Chunker
+          # Trace agent limit payload size of 10 MiB (since agent v5.11.0):
+          # https://github.com/DataDog/datadog-agent/blob/6.14.1/pkg/trace/api/api.go#L46
+          #
+          # We set the value to a conservative 5 MiB, in case network speed is slow.
+          DEFAULT_MAX_PAYLOAD_SIZE = 5 * 1024 * 1024
+
+          attr_reader :encoder, :max_size
+
+          #
+          # Single traces larger than +max_size+ will be discarded.
+          #
+          # @param encoder [Datadog::Core::Encoding::Encoder]
+          # @param max_size [String] maximum acceptable payload size
+          def initialize(encoder, max_size: DEFAULT_MAX_PAYLOAD_SIZE)
+            @encoder = encoder
+            @max_size = max_size
+          end
+
+          # Encodes a list of traces in chunks.
+          # Before serializing, all traces are normalized. Trace nesting is not changed.
+          #
+          # @param traces [Enumerable<Trace>] list of traces
+          # @return [Enumerable[Array[Bytes,Integer]]] list of encoded chunks: each containing a byte array and
+          #   number of traces
+          def encode_in_chunks(traces)
+            encoded_traces = if traces.respond_to?(:filter_map)
+                               # DEV Supported since Ruby 2.7, saves an intermediate object creation
+                               traces.filter_map { |t| encode_one(t) }
+                             else
+                               traces.map { |t| encode_one(t) }.reject(&:nil?)
+                             end
+
+            Datadog::Core::Chunker.chunk_by_size(encoded_traces, max_size).map do |chunk|
+              [encoder.join(chunk), chunk.size]
+            end
+          end
+
+          private
+
+          def encode_one(trace)
+            encoded = Encoder.encode_trace(encoder, trace)
+
+            if encoded.size > max_size
+              # This single trace is too large, we can't flush it
+              Datadog.logger.debug { "Dropping trace. Payload too large: '#{trace.inspect}'" }
+              Datadog.health_metrics.transport_trace_too_large(1)
+
+              return nil
+            end
+
+            encoded
+          end
+        end
+
+        # Encodes traces using {Datadog::Core::Encoding::Encoder} instances.
+        module Encoder
+          module_function
+
+          def encode_trace(encoder, trace)
+            # Format the trace for transport
+            TraceFormatter.format!(trace)
+
+            # Make the trace serializable
+            serializable_trace = SerializableTrace.new(trace)
+
+            Datadog.logger.debug { "Flushing trace: #{JSON.dump(serializable_trace)}" }
+
+            # Encode the trace
+            encoder.encode(serializable_trace)
+          end
+        end
+
+        # Sends traces based on transport API configuration.
+        #
+        # This class initializes the HTTP client, breaks down large
+        # batches of traces into smaller chunks and handles
+        # API version downgrade handshake.
+        class Transport
+          attr_reader :client, :apis, :default_api, :current_api_id
+
+          def initialize(apis, default_api)
+            @apis = apis
+            @default_api = default_api
+
+            change_api!(default_api)
+          end
+
+          def send_traces(traces)
+            encoder = current_api.encoder
+            chunker = Datadog::Tracing::Transport::Traces::Chunker.new(encoder)
+
+            responses = chunker.encode_in_chunks(traces.lazy).map do |encoded_traces, trace_count|
+              request = Request.new(EncodedParcel.new(encoded_traces, trace_count))
+
+              client.send_traces_payload(request).tap do |response|
+                if downgrade?(response)
+                  downgrade!
+                  return send_traces(traces)
+                end
+              end
+            end
+
+            # Force resolution of lazy enumerator.
+            #
+            # The "correct" method to call here would be `#force`,
+            # as this method was created to force the eager loading
+            # of a lazy enumerator.
+            #
+            # Unfortunately, JRuby < 9.2.9.0 erroneously eagerly loads
+            # the lazy Enumerator during intermediate steps.
+            # This forces us to use `#to_a`, as this method works for both
+            # lazy and regular Enumerators.
+            # Using `#to_a` can mask the fact that we expect a lazy
+            # Enumerator.
+            responses = responses.to_a
+
+            Datadog.health_metrics.transport_chunked(responses.size)
+
+            responses
+          end
+
+          def stats
+            @client.stats
+          end
+
+          def current_api
+            apis[@current_api_id]
+          end
+
+          private
+
+          def downgrade?(response)
+            return false unless apis.fallbacks.key?(@current_api_id)
+
+            response.not_found? || response.unsupported?
+          end
+
+          def downgrade!
+            downgrade_api_id = apis.fallbacks[@current_api_id]
+            raise NoDowngradeAvailableError, @current_api_id if downgrade_api_id.nil?
+
+            change_api!(downgrade_api_id)
+          end
+
+          def change_api!(api_id)
+            raise UnknownApiVersionError, api_id unless apis.key?(api_id)
+
+            @current_api_id = api_id
+            @client = HTTP::Client.new(current_api)
+          end
+
+          # Raised when configured with an unknown API version
+          class UnknownApiVersionError < StandardError
+            attr_reader :version
+
+            def initialize(version)
+              super
+
+              @version = version
+            end
+
+            def message
+              "No matching transport API for version #{version}!"
+            end
+          end
+
+          # Raised when configured with an unknown API version
+          class NoDowngradeAvailableError < StandardError
+            attr_reader :version
+
+            def initialize(version)
+              super
+
+              @version = version
+            end
+
+            def message
+              "No downgrade from transport API version #{version} is available!"
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/utils.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require_relative '../core/utils/forking'
+require_relative '../core/utils/time'
+
+module Datadog
+  module Tracing
+    # Utils contains low-level tracing utility functions.
+    # @public_api
+    module Utils
+      extend Datadog::Core::Utils::Forking
+
+      # The max value for a {Datadog::Tracing::Span} identifier.
+      # Span and trace identifiers should be strictly positive and strictly inferior to this limit.
+      #
+      # Limited to +2<<62-1+ positive integers, as Ruby is able to represent such numbers "inline",
+      # inside a +VALUE+ scalar, thus not requiring memory allocation.
+      #
+      # The range of IDs also has to consider portability across different languages and platforms.
+      RUBY_MAX_ID = (1 << 62) - 1
+
+      # Excludes zero from possible values
+      RUBY_ID_RANGE = (1..RUBY_MAX_ID).freeze
+
+      # While we only generate 63-bit integers due to limitations in other languages, we support
+      # parsing 64-bit integers for distributed tracing since an upstream system may generate one
+      EXTERNAL_MAX_ID = 1 << 64
+
+      # We use a custom random number generator because we want no interference
+      # with the default one. Using the default prng, we could break code that
+      # would rely on srand/rand sequences.
+
+      # Return a randomly generated integer, valid as a Span ID or Trace ID.
+      # This method is thread-safe and fork-safe.
+      def self.next_id
+        after_fork! { reset! }
+        id_rng.rand(RUBY_ID_RANGE)
+      end
+
+      def self.id_rng
+        @id_rng ||= Random.new
+      end
+
+      def self.reset!
+        @id_rng = Random.new
+      end
+
+      private_class_method :id_rng, :reset!
+
+      # The module handles bitwise operation for trace id
+      module TraceId
+        MAX = (1 << 128) - 1
+
+        module_function
+
+        # Format for generating 128 bits trace id =>
+        # - 32-bits : seconds since Epoch
+        # - 32-bits : set to zero,
+        # - 64 bits : random 64-bits
+        def next_id
+          return Utils.next_id unless Datadog.configuration.tracing.trace_id_128_bit_generation_enabled
+
+          concatenate(
+            Core::Utils::Time.now.to_i << 32,
+            Utils.next_id
+          )
+        end
+
+        def to_high_order(trace_id)
+          trace_id >> 64
+        end
+
+        def to_low_order(trace_id)
+          trace_id & 0xFFFFFFFFFFFFFFFF
+        end
+
+        def concatenate(high_order, low_order)
+          high_order << 64 | low_order
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/workers/trace_writer.rb

@@ -1,6 +1,3 @@
-# typed: false
-
-require_relative '../../core'
 require_relative '../../core/worker'
 require_relative '../../core/workers/async'
 require_relative '../../core/workers/polling'
@@ -10,7 +7,7 @@ require_relative '../buffer'
 require_relative '../pipeline'
 require_relative '../event'
 
-require_relative '../../../ddtrace/transport/http'
+require_relative '../transport/http'
 
 module Datadog
   module Tracing
@@ -27,7 +24,7 @@ module Datadog
           transport_options[:agent_settings] = options[:agent_settings] if options.key?(:agent_settings)
 
           @transport = options.fetch(:transport) do
-            Transport::HTTP.default(**transport_options)
+            Datadog::Tracing::Transport::HTTP.default(**transport_options)
           end
         end
         # rubocop:enable Lint/MissingSuper
@@ -60,7 +57,7 @@ module Datadog
           end
         end
 
-        # TODO: Register `Datadog::Core::Diagnostics::EnvironmentLogger.log!`
+        # TODO: Register `Datadog::Tracing::Diagnostics::EnvironmentLogger.collect_and_log!`
         # TODO: as a flush_completed subscriber when the `TraceWriter`
         # TODO: instantiation code is implemented.
         def flush_completed
@@ -107,6 +104,8 @@ module Datadog
           # Workers::Queue settings
           @buffer_size = options.fetch(:buffer_size, DEFAULT_BUFFER_MAX_SIZE)
           self.buffer = TraceBuffer.new(@buffer_size)
+
+          @shutdown_timeout = options.fetch(:shutdown_timeout, Core::Workers::Polling::DEFAULT_SHUTDOWN_TIMEOUT)
         end
 
         # NOTE: #perform is wrapped by other modules:
@@ -122,7 +121,7 @@ module Datadog
           nil
         end
 
-        def stop(*args)
+        def stop(force_stop = false, timeout = @shutdown_timeout)
           buffer.close if running?
           super
         end

data/lib/datadog/tracing/workers.rb

@@ -1,7 +1,3 @@
-# typed: true
-
-require_relative '../core'
-
 require_relative 'buffer'
 require_relative 'pipeline'
 
@@ -18,7 +14,7 @@ module Datadog
         DEFAULT_TIMEOUT = 5
         BACK_OFF_RATIO = 1.2
         BACK_OFF_MAX = 5
-        SHUTDOWN_TIMEOUT = 1
+        DEFAULT_SHUTDOWN_TIMEOUT = 1
 
         attr_reader \
           :trace_buffer
@@ -40,6 +36,7 @@ module Datadog
 
           # Threading
           @shutdown = ConditionVariable.new
+          @shutdown_timeout = options.fetch(:shutdown_timeout, DEFAULT_SHUTDOWN_TIMEOUT)
           @mutex = Mutex.new
           @worker = nil
           @run = false
@@ -72,6 +69,7 @@ module Datadog
             Datadog.logger.debug { "Starting thread for: #{self}" }
             @worker = Thread.new { perform }
             @worker.name = self.class.name unless Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.3')
+            @worker.thread_variable_set(:fork_safe, true)
 
             nil
           end
@@ -93,7 +91,7 @@ module Datadog
 
         # Block until executor shutdown is complete or until timeout seconds have passed.
         def join
-          @worker.join(SHUTDOWN_TIMEOUT)
+          @worker.join(@shutdown_timeout)
         end
 
         # Enqueue an item in the trace internal buffer. This operation is thread-safe

data/lib/datadog/tracing/writer.rb

@@ -1,12 +1,8 @@
-# typed: true
-
-require_relative '../core'
-
 require_relative 'event'
 require_relative 'runtime/metrics'
 require_relative 'workers'
 
-require_relative '../../ddtrace/transport/http'
+require_relative 'transport/http'
 
 module Datadog
   module Tracing
@@ -32,6 +28,8 @@ module Datadog
           Transport::HTTP.default(**transport_options)
         end
 
+        @shutdown_timeout = options.fetch(:shutdown_timeout, Workers::AsyncTransport::DEFAULT_SHUTDOWN_TIMEOUT)
+
         # handles the thread creation after an eventual fork
         @mutex_after_fork = Mutex.new
         @pid = nil
@@ -76,7 +74,8 @@ module Datadog
           transport: @transport,
           buffer_size: @buff_size,
           on_trace: @trace_handler,
-          interval: @flush_interval
+          interval: @flush_interval,
+          shutdown_timeout: @shutdown_timeout
         )
 
         @worker.start
@@ -175,6 +174,13 @@ module Datadog
           end
         end
       end
+
+      private
+
+      def reset_stats!
+        @traces_flushed = 0
+        @transport.stats.reset!
+      end
     end
   end
 end

data/lib/datadog/tracing.rb

@@ -1,4 +1,4 @@
-# typed: false
+# frozen_string_literal: true
 
 require_relative 'core'
 require_relative 'tracing/pipeline'

data/lib/ddtrace/auto_instrument.rb

@@ -1,4 +1,4 @@
-# typed: strict
+# frozen_string_literal: true
 
 # Entrypoint file for auto instrumentation.
 #

data/lib/ddtrace/auto_instrument_base.rb

@@ -1,4 +1,4 @@
-# typed: true
+# frozen_string_literal: true
 
 module Datadog
   # base methods stubbed for adding auto instrument extensions

data/lib/ddtrace/profiling/preload.rb

@@ -1,4 +1,2 @@
-# typed: strict
-
 # TODO: Warns users after 1.0 upgrade. Remove this file in 2.0.
 raise LoadError, 'This file has been moved to `datadog/profiling/preload` in 1.0.0'

data/lib/ddtrace/transport/ext.rb

@@ -1,40 +1,46 @@
-# typed: true
+# frozen_string_literal: true
 
+require_relative '../../datadog/core/transport/ext'
+# DEV(2.0): This file should be replaced by /datadog/core/transport/ext.rb.
 module Datadog
   module Transport
     # @public_api
     module Ext
       # @public_api
       module HTTP
-        ADAPTER = :net_http # DEV: Rename to simply `:http`, as Net::HTTP is an implementation detail.
-        DEFAULT_HOST = '127.0.0.1'.freeze
-        DEFAULT_PORT = 8126
+        ADAPTER = Datadog::Core::Transport::Ext::HTTP::ADAPTER
+        DEFAULT_HOST = Datadog::Core::Transport::Ext::HTTP::DEFAULT_HOST
+        DEFAULT_PORT = Datadog::Core::Transport::Ext::HTTP::DEFAULT_PORT
 
-        HEADER_CONTAINER_ID = 'Datadog-Container-ID'.freeze
-        HEADER_DD_API_KEY = 'DD-API-KEY'.freeze
+        HEADER_CONTAINER_ID = Datadog::Core::Transport::Ext::HTTP::HEADER_CONTAINER_ID
+        HEADER_DD_API_KEY = Datadog::Core::Transport::Ext::HTTP::HEADER_DD_API_KEY
         # Tells agent that `_dd.top_level` metrics have been set by the tracer.
         # The agent will not calculate top-level spans but instead trust the tracer tagging.
         #
         # This prevents partially flushed traces being mistakenly marked as top-level.
         #
         # Setting this header to any non-empty value enables this feature.
-        HEADER_CLIENT_COMPUTED_TOP_LEVEL = 'Datadog-Client-Computed-Top-Level'.freeze
-        HEADER_META_LANG = 'Datadog-Meta-Lang'.freeze
-        HEADER_META_LANG_VERSION = 'Datadog-Meta-Lang-Version'.freeze
-        HEADER_META_LANG_INTERPRETER = 'Datadog-Meta-Lang-Interpreter'.freeze
-        HEADER_META_TRACER_VERSION = 'Datadog-Meta-Tracer-Version'.freeze
+        HEADER_CLIENT_COMPUTED_TOP_LEVEL = Datadog::Core::Transport::Ext::HTTP::HEADER_CLIENT_COMPUTED_TOP_LEVEL
+        HEADER_META_LANG = Datadog::Core::Transport::Ext::HTTP::HEADER_META_LANG
+        HEADER_META_LANG_VERSION = Datadog::Core::Transport::Ext::HTTP::HEADER_META_LANG_VERSION
+        HEADER_META_LANG_INTERPRETER = Datadog::Core::Transport::Ext::HTTP::HEADER_META_LANG_INTERPRETER
+        HEADER_META_TRACER_VERSION = Datadog::Core::Transport::Ext::HTTP::HEADER_META_TRACER_VERSION
+
+        # Header that prevents the Net::HTTP integration from tracing internal trace requests.
+        # Set it to any value to skip tracing.
+        HEADER_DD_INTERNAL_UNTRACED_REQUEST = Datadog::Core::Transport::Ext::HTTP::HEADER_DD_INTERNAL_UNTRACED_REQUEST
       end
 
       # @public_api
       module Test
-        ADAPTER = :test
+        ADAPTER = Datadog::Core::Transport::Ext::Test::ADAPTER
       end
 
       # @public_api
       module UnixSocket
-        ADAPTER = :unix
-        DEFAULT_PATH = '/var/run/datadog/apm.socket'.freeze
-        DEFAULT_TIMEOUT_SECONDS = 1
+        ADAPTER = Datadog::Core::Transport::Ext::UnixSocket::ADAPTER
+        DEFAULT_PATH = Datadog::Core::Transport::Ext::UnixSocket::DEFAULT_PATH
+        DEFAULT_TIMEOUT_SECONDS = Datadog::Core::Transport::Ext::UnixSocket::DEFAULT_TIMEOUT_SECONDS
       end
     end
   end

data/lib/ddtrace/version.rb

@@ -1,27 +1,26 @@
-# typed: true
+# frozen_string_literal: true
 
 module DDTrace
   module VERSION
     MAJOR = 1
-    MINOR = 4
-    PATCH = 2
+    MINOR = 19
+    PATCH = 0
     PRE = nil
+    BUILD = nil
+    # PRE and BUILD above are modified for dev gems during gem build GHA workflow
 
-    STRING = [MAJOR, MINOR, PATCH, PRE].compact.join('.')
+    STRING = [MAJOR, MINOR, PATCH, PRE, BUILD].compact.join('.')
 
-    MINIMUM_RUBY_VERSION = '2.1.0'.freeze
+    MINIMUM_RUBY_VERSION = '2.1.0'
 
-    # A maximum version was initially added in https://github.com/DataDog/dd-trace-rb/pull/1495 because we expected
-    # the `ruby2_keywords` method to be removed (see the PR for the discussion).
-    # That is because Ruby 3.x support as implemented using `*args` needs `ruby2_keywords` to continue working,
-    # but if `ruby2_keywords` gets removed we would need to change the code to use `*args, **kwargs`.
+    # Restrict the installation of this gem with untested future versions of Ruby.
     #
-    # Now Ruby 3.2.0-preview1 is out and `ruby2_keywords` are still there, and there's even a recent change for it
-    # in https://github.com/ruby/ruby/pull/5684 that is documented as "ruby2_keywords needed in 3.2+".
+    # This prevents crashes in the native extension code and sends a clear signal to the
+    # user that this version of the gem is untested for the host Ruby version.
     #
-    # So for now let's bump the maximum version to < 3.3 to allow the Ruby 3.2 series to be supported and we can keep
-    # an eye on the Ruby 3.2 test releases to see if anything changes. (Otherwise, once Ruby 3.2.0 stable is out, we
-    # should probably bump this to 3.4, and so on...)
-    MAXIMUM_RUBY_VERSION = '3.3'.freeze
+    # To allow testing with the next unreleased version of Ruby, the version check is performed
+    # as `< #{MAXIMUM_RUBY_VERSION}`, meaning prereleases of MAXIMUM_RUBY_VERSION are allowed
+    # but not stable MAXIMUM_RUBY_VERSION releases.
+    MAXIMUM_RUBY_VERSION = '3.4'
   end
 end

data/lib/ddtrace.rb

@@ -1,13 +1,11 @@
-# typed: strict
+# frozen_string_literal: true
 
 # Load tracing
 require_relative 'datadog/tracing'
 require_relative 'datadog/tracing/contrib'
 
-# Load appsec
-require_relative 'datadog/appsec/autoload' # TODO: datadog/appsec?
-
 # Load other products (must follow tracing)
 require_relative 'datadog/profiling'
-require_relative 'datadog/ci'
+require_relative 'datadog/appsec'
+require 'datadog/ci'
 require_relative 'datadog/kit'