ddtrace 0.35.2 → 1.1.0

data/lib/ddtrace/transport/io/traces.rb

@@ -1,3 +1,5 @@
+# typed: false
+
 require 'ddtrace/transport/traces'
 
 require 'ddtrace/transport/io/response'
@@ -10,10 +12,18 @@ module Datadog
       module Traces
         # Response from HTTP transport for traces
         class Response < IO::Response
+          include Transport::Traces::Response
+
+          def initialize(result, trace_count = 1)
+            super(result)
+            @trace_count = trace_count
+          end
         end
 
         # Extensions for HTTP client
         module Client
+          include Kernel # Ensure that kernel methods are always available (https://sorbet.org/docs/error-reference#7003)
+
           def send_traces(traces)
             # Build a request
             req = Transport::Traces::Request.new(Parcel.new(traces))
@@ -58,7 +68,7 @@ module Datadog
 
           def encode_trace(trace)
             # Convert each trace to hash
-            trace.map(&:to_hash).tap do |spans|
+            trace.spans.map(&:to_hash).tap do |spans|
               # Convert IDs to hexadecimal
               spans.each do |span|
                 ENCODED_IDS.each do |id|
@@ -84,7 +94,7 @@ module Datadog
         end
 
         # Add traces behavior to transport components
-        IO::Client.send(:include, Traces::Client)
+        IO::Client.include(Traces::Client)
       end
     end
   end

data/lib/ddtrace/transport/io.rb

@@ -1,4 +1,6 @@
-require 'ddtrace/encoding'
+# typed: true
+
+require 'datadog/core/encoding'
 require 'ddtrace/transport/io/client'
 require 'ddtrace/transport/io/traces'
 
@@ -17,8 +19,8 @@ module Datadog
       # Pass options to override any settings.
       def default(options = {})
         new(
-          options.fetch(:out, STDOUT),
-          options.fetch(:encoder, Encoding::JSONEncoder)
+          options.fetch(:out, $stdout),
+          options.fetch(:encoder, Core::Encoding::JSONEncoder)
         )
       end
     end

data/lib/ddtrace/transport/parcel.rb

@@ -1,13 +1,22 @@
+# typed: true
+
 module Datadog
   module Transport
     # Data transfer object for generic data
+    # @abstract
     module Parcel
+      include Kernel # Ensure that kernel methods are always available (https://sorbet.org/docs/error-reference#7003)
+
       attr_reader \
         :data
 
       def initialize(data)
         @data = data
       end
+
+      def encode_with(encoder)
+        raise NotImplementedError
+      end
     end
   end
 end

data/lib/ddtrace/transport/request.rb

@@ -1,3 +1,5 @@
+# typed: true
+
 module Datadog
   module Transport
     # Defines request for transport operations

data/lib/ddtrace/transport/response.rb

@@ -1,3 +1,5 @@
+# typed: false
+
 module Datadog
   module Transport
     # Defines abstract response for transport operations
@@ -29,6 +31,13 @@ module Datadog
       def internal_error?
         nil
       end
+
+      def inspect
+        "#{self.class} ok?:#{ok?} unsupported?:#{unsupported?}, " \
+        "not_found?:#{not_found?}, client_error?:#{client_error?}, " \
+        "server_error?:#{server_error?}, internal_error?:#{internal_error?}, " \
+        "payload:#{payload}"
+      end
     end
 
     # A generic error response for internal errors
@@ -44,6 +53,10 @@ module Datadog
       def internal_error?
         true
       end
+
+      def inspect
+        "#{super}, error_type:#{error.class} error:#{error}"
+      end
     end
   end
 end

data/lib/ddtrace/transport/serializable_trace.rb

@@ -0,0 +1,118 @@
+# typed: true
+
+require 'json'
+require 'msgpack'
+
+module Datadog
+  module Transport
+    # Adds serialization functions to a {Datadog::TraceSegment}
+    class SerializableTrace
+      attr_reader \
+        :trace
+
+      def initialize(trace)
+        @trace = trace
+      end
+
+      # MessagePack serializer interface. Making this object
+      # respond to `#to_msgpack` allows it to be automatically
+      # serialized by MessagePack.
+      #
+      # This is more efficient than doing +MessagePack.pack(span.to_hash)+
+      # as we don't have to create an intermediate Hash.
+      #
+      # @param packer [MessagePack::Packer] serialization buffer, can be +nil+ with JRuby
+      def to_msgpack(packer = nil)
+        # As of 1.3.3, JRuby implementation doesn't pass an existing packer
+        trace.spans.map { |s| SerializableSpan.new(s) }.to_msgpack(packer)
+      end
+
+      # JSON serializer interface.
+      # Used by older version of the transport.
+      def to_json(*args)
+        trace.spans.map(&:to_hash).to_json(*args)
+      end
+    end
+
+    # Adds serialization functions to a {Datadog::Span}
+    class SerializableSpan
+      attr_reader \
+        :span
+
+      def initialize(span)
+        @span = span
+      end
+
+      # MessagePack serializer interface. Making this object
+      # respond to `#to_msgpack` allows it to be automatically
+      # serialized by MessagePack.
+      #
+      # This is more efficient than doing +MessagePack.pack(span.to_hash)+
+      # as we don't have to create an intermediate Hash.
+      #
+      # @param packer [MessagePack::Packer] serialization buffer, can be +nil+ with JRuby
+      # rubocop:disable Metrics/AbcSize
+      def to_msgpack(packer = nil)
+        packer ||= MessagePack::Packer.new
+
+        number_of_elements_to_write = 10
+
+        if span.stopped?
+          packer.write_map_header(number_of_elements_to_write + 2) # Set header with how many elements in the map
+
+          packer.write('start')
+          packer.write(time_nano(span.start_time))
+
+          packer.write('duration')
+          packer.write(duration_nano(span.duration))
+        else
+          packer.write_map_header(number_of_elements_to_write) # Set header with how many elements in the map
+        end
+
+        # DEV: We use strings as keys here, instead of symbols, as
+        # DEV: MessagePack will ultimately convert them to strings.
+        # DEV: By providing strings directly, we skip this indirection operation.
+        packer.write('span_id')
+        packer.write(span.id)
+        packer.write('parent_id')
+        packer.write(span.parent_id)
+        packer.write('trace_id')
+        packer.write(span.trace_id)
+        packer.write('name')
+        packer.write(span.name)
+        packer.write('service')
+        packer.write(span.service)
+        packer.write('resource')
+        packer.write(span.resource)
+        packer.write('type')
+        packer.write(span.type)
+        packer.write('meta')
+        packer.write(span.meta)
+        packer.write('metrics')
+        packer.write(span.metrics)
+        packer.write('error')
+        packer.write(span.status)
+        packer
+      end
+      # rubocop:enable Metrics/AbcSize
+
+      # JSON serializer interface.
+      # Used by older version of the transport.
+      def to_json(*args)
+        span.to_hash.to_json(*args)
+      end
+
+      # Used for serialization
+      # @return [Integer] in nanoseconds since Epoch
+      def time_nano(time)
+        time.to_i * 1000000000 + time.nsec
+      end
+
+      # Used for serialization
+      # @return [Integer] in nanoseconds since Epoch
+      def duration_nano(duration)
+        (duration * 1e9).to_i
+      end
+    end
+  end
+end

data/lib/ddtrace/transport/statistics.rb

@@ -1,4 +1,7 @@
-require 'ddtrace/diagnostics/health'
+# typed: true
+
+require 'datadog/core/metrics/metric'
+require 'datadog/core/diagnostics/health'
 
 module Datadog
   module Transport
@@ -27,8 +30,8 @@ module Datadog
 
       def metrics_for_response(response)
         {}.tap do |metrics|
-          metrics[:api_errors] = Metrics::Metric.new(:api_errors, nil, 1) if response.internal_error?
-          metrics[:api_responses] = Metrics::Metric.new(:api_responses, nil, 1) unless response.internal_error?
+          metrics[:api_errors] = Core::Metrics::Metric.new(:api_errors, nil, 1) if response.internal_error?
+          metrics[:api_responses] = Core::Metrics::Metric.new(:api_responses, nil, 1) unless response.internal_error?
         end
       end
 
@@ -43,7 +46,7 @@ module Datadog
       end
 
       def metrics_for_exception(_exception)
-        { api_errors: Metrics::Metric.new(:api_errors, nil, 1) }
+        { api_errors: Core::Metrics::Metric.new(:api_errors, nil, 1) }
       end
 
       # Stat counts

data/lib/ddtrace/transport/trace_formatter.rb

@@ -0,0 +1,187 @@
+# typed: true
+
+require 'datadog/core/environment/identity'
+require 'datadog/core/environment/socket'
+require 'datadog/core/runtime/ext'
+require 'datadog/tracing/metadata/ext'
+require 'datadog/tracing/trace_segment'
+
+module Datadog
+  module Transport
+    # Prepares traces for transport
+    class TraceFormatter
+      attr_reader \
+        :root_span,
+        :trace
+
+      def self.format!(trace)
+        new(trace).format!
+      end
+
+      def initialize(trace)
+        @trace = trace
+        @root_span = find_root_span(trace)
+      end
+
+      # Modifies a trace so suitable for transport
+      def format!
+        return unless trace
+        return trace unless root_span
+
+        # Because the trace API does not support
+        # trace metadata, we must put our trace
+        # metadata on the root span. This "root span"
+        # is needed by the agent/API to ingest the trace.
+
+        # Apply generic trace tags. Any more specific value will be overridden
+        # by the subsequent calls below.
+        set_trace_tags!
+
+        set_resource!
+
+        tag_agent_sample_rate!
+        tag_hostname!
+        tag_lang!
+        tag_origin!
+        tag_process_id!
+        tag_rule_sample_rate!
+        tag_runtime_id!
+        tag_rate_limiter_rate!
+        tag_sample_rate!
+        tag_sampling_priority!
+
+        trace
+      end
+
+      protected
+
+      def set_resource!
+        # If the trace resource is undefined, or the root span wasn't
+        # specified, don't set this. We don't want to overwrite the
+        # resource of a span that is in the middle of the trace.
+        return if trace.resource.nil? || partial?
+
+        root_span.resource = trace.resource
+      end
+
+      def set_trace_tags!
+        # If the root span wasn't specified, don't set this. We don't want to
+        # misset or overwrite the tags of a span that is in the middle of the
+        # trace.
+        return if partial?
+
+        root_span.set_tags(trace.send(:meta))
+        root_span.set_tags(trace.send(:metrics))
+      end
+
+      def tag_agent_sample_rate!
+        return unless trace.agent_sample_rate
+
+        root_span.set_tag(
+          Tracing::Metadata::Ext::Sampling::TAG_AGENT_RATE,
+          trace.agent_sample_rate
+        )
+      end
+
+      def tag_hostname!
+        return unless trace.hostname
+
+        root_span.set_tag(
+          Tracing::Metadata::Ext::NET::TAG_HOSTNAME,
+          trace.hostname
+        )
+      end
+
+      def tag_lang!
+        return if trace.lang.nil? || root_span.get_tag(Tracing::Metadata::Ext::TAG_PEER_SERVICE)
+
+        root_span.set_tag(
+          Core::Runtime::Ext::TAG_LANG,
+          trace.lang
+        )
+      end
+
+      def tag_origin!
+        return unless trace.origin
+
+        root_span.set_tag(
+          Tracing::Metadata::Ext::Distributed::TAG_ORIGIN,
+          trace.origin
+        )
+      end
+
+      def tag_process_id!
+        return unless trace.process_id
+
+        root_span.set_tag(
+          Core::Runtime::Ext::TAG_PID,
+          trace.process_id
+        )
+      end
+
+      def tag_rate_limiter_rate!
+        return unless trace.rate_limiter_rate
+
+        root_span.set_tag(
+          Tracing::Metadata::Ext::Sampling::TAG_RATE_LIMITER_RATE,
+          trace.rate_limiter_rate
+        )
+      end
+
+      def tag_rule_sample_rate!
+        return unless trace.rule_sample_rate
+
+        root_span.set_tag(
+          Tracing::Metadata::Ext::Sampling::TAG_RULE_SAMPLE_RATE,
+          trace.rule_sample_rate
+        )
+      end
+
+      def tag_runtime_id!
+        return unless trace.runtime_id
+
+        root_span.set_tag(
+          Core::Runtime::Ext::TAG_ID,
+          trace.runtime_id
+        )
+      end
+
+      def tag_sample_rate!
+        return unless trace.sample_rate
+
+        root_span.set_tag(
+          Tracing::Metadata::Ext::Sampling::TAG_SAMPLE_RATE,
+          trace.sample_rate
+        )
+      end
+
+      def tag_sampling_priority!
+        return unless trace.sampling_priority
+
+        root_span.set_metric(
+          Tracing::Metadata::Ext::Distributed::TAG_SAMPLING_PRIORITY,
+          trace.sampling_priority
+        )
+      end
+
+      private
+
+      def partial?
+        !@found_root_span
+      end
+
+      def find_root_span(trace)
+        # TODO: Should we memoize this?
+        #       Would be safe, but `spans` is mutable, so if
+        #       the root span were removed, it would be a stale reference.
+        #       Figure out a better way to deal with this.
+        root_span_id = trace.send(:root_span_id)
+        root_span = trace.spans.find { |s| s.id == root_span_id } if root_span_id
+        @found_root_span = !root_span.nil?
+
+        # when root span is not found, fall back to last span (partial flush)
+        root_span || trace.spans.last
+      end
+    end
+  end
+end

data/lib/ddtrace/transport/traces.rb

@@ -1,13 +1,17 @@
+# typed: true
+
+require 'datadog/core/chunker'
 require 'ddtrace/transport/parcel'
 require 'ddtrace/transport/request'
-require 'ddtrace/chunker'
+require 'ddtrace/transport/serializable_trace'
+require 'ddtrace/transport/trace_formatter'
 
 module Datadog
   module Transport
     module Traces
       # Data transfer object for encoded traces
       class EncodedParcel
-        include Transport::Parcel
+        include Datadog::Transport::Parcel
 
         attr_reader :trace_count
 
@@ -22,7 +26,7 @@ module Datadog
       end
 
       # Traces request
-      class Request < Transport::Request
+      class Request < Datadog::Transport::Request
       end
 
       # Traces response
@@ -43,7 +47,7 @@ module Datadog
         #
         # Single traces larger than +max_size+ will be discarded.
         #
-        # @param encoder [Datadog::Encoding::Encoder]
+        # @param encoder [Datadog::Core::Encoding::Encoder]
         # @param max_size [String] maximum acceptable payload size
         def initialize(encoder, max_size: DEFAULT_MAX_PAYLOAD_SIZE)
           @encoder = encoder
@@ -57,9 +61,14 @@ module Datadog
         # @return [Enumerable[Array[Bytes,Integer]]] list of encoded chunks: each containing a byte array and
         #   number of traces
         def encode_in_chunks(traces)
-          encoded_traces = traces.map { |t| encode_one(t) }.reject(&:nil?)
-
-          Datadog::Chunker.chunk_by_size(encoded_traces, max_size).map do |chunk|
+          encoded_traces = if traces.respond_to?(:filter_map)
+                             # DEV Supported since Ruby 2.7, saves an intermediate object creation
+                             traces.filter_map { |t| encode_one(t) }
+                           else
+                             traces.map { |t| encode_one(t) }.reject(&:nil?)
+                           end
+
+          Datadog::Core::Chunker.chunk_by_size(encoded_traces, max_size).map do |chunk|
             [encoder.join(chunk), chunk.size]
           end
         end
@@ -81,12 +90,19 @@ module Datadog
         end
       end
 
-      # Encodes traces using {Datadog::Encoding::Encoder} instances.
+      # Encodes traces using {Datadog::Core::Encoding::Encoder} instances.
       module Encoder
         module_function
 
         def encode_trace(encoder, trace)
-          encoder.encode(trace.map(&:to_hash))
+          # Format the trace for transport
+          TraceFormatter.format!(trace)
+
+          # Make the trace serializable
+          serializable_trace = SerializableTrace.new(trace)
+
+          # Encode the trace
+          encoder.encode(serializable_trace)
         end
       end
 
@@ -118,7 +134,21 @@ module Datadog
                 return send_traces(traces)
               end
             end
-          end.force
+          end
+
+          # Force resolution of lazy enumerator.
+          #
+          # The "correct" method to call here would be `#force`,
+          # as this method was created to force the eager loading
+          # of a lazy enumerator.
+          #
+          # Unfortunately, JRuby < 9.2.9.0 erroneously eagerly loads
+          # the lazy Enumerator during intermediate steps.
+          # This forces us to use `#to_a`, as this method works for both
+          # lazy and regular Enumerators.
+          # Using `#to_a` can mask the fact that we expect a lazy
+          # Enumerator.
+          responses = responses.to_a
 
           Datadog.health_metrics.transport_chunked(responses.size)
 
@@ -137,17 +167,20 @@ module Datadog
 
         def downgrade?(response)
           return false unless apis.fallbacks.key?(@current_api_id)
+
           response.not_found? || response.unsupported?
         end
 
         def downgrade!
           downgrade_api_id = apis.fallbacks[@current_api_id]
           raise NoDowngradeAvailableError, @current_api_id if downgrade_api_id.nil?
+
           change_api!(downgrade_api_id)
         end
 
         def change_api!(api_id)
           raise UnknownApiVersionError, api_id unless apis.key?(api_id)
+
           @current_api_id = api_id
           @client = HTTP::Client.new(current_api)
         end

data/lib/ddtrace/version.rb

@@ -1,12 +1,27 @@
-module Datadog
+# typed: true
+
+module DDTrace
   module VERSION
-    MAJOR = 0
-    MINOR = 35
-    PATCH = 2
+    MAJOR = 1
+    MINOR = 1
+    PATCH = 0
     PRE = nil
 
     STRING = [MAJOR, MINOR, PATCH, PRE].compact.join('.')
 
-    MINIMUM_RUBY_VERSION = '2.0.0'.freeze
+    MINIMUM_RUBY_VERSION = '2.1.0'.freeze
+
+    # A maximum version was initially added in https://github.com/DataDog/dd-trace-rb/pull/1495 because we expected
+    # the `ruby2_keywords` method to be removed (see the PR for the discussion).
+    # That is because Ruby 3.x support as implemented using `*args` needs `ruby2_keywords` to continue working,
+    # but if `ruby2_keywords` gets removed we would need to change the code to use `*args, **kwargs`.
+    #
+    # Now Ruby 3.2.0-preview1 is out and `ruby2_keywords` are still there, and there's even a recent change for it
+    # in https://github.com/ruby/ruby/pull/5684 that is documented as "ruby2_keywords needed in 3.2+".
+    #
+    # So for now let's bump the maximum version to < 3.3 to allow the Ruby 3.2 series to be supported and we can keep
+    # an eye on the Ruby 3.2 test releases to see if anything changes. (Otherwise, once Ruby 3.2.0 stable is out, we
+    # should probably bump this to 3.4, and so on...)
+    MAXIMUM_RUBY_VERSION = '3.3'.freeze
   end
 end

data/lib/ddtrace.rb

@@ -1,74 +1,13 @@
-require 'thread'
+# typed: strict
 
-# During development, we load `ddtrace` by through `ddtrace.gemspec`,
-# which in turn eager loads 'ddtrace/version'.
-#
-# Users load this gem by requiring this file.
-# We need to ensure that any files loaded in our gemspec are also loaded here.
-require 'ddtrace/version'
+# Load tracing
+require 'datadog/tracing'
+require 'datadog/tracing/contrib'
 
-require 'ddtrace/pin'
-require 'ddtrace/tracer'
-require 'ddtrace/error'
-require 'ddtrace/quantization/hash'
-require 'ddtrace/quantization/http'
-require 'ddtrace/pipeline'
-require 'ddtrace/configuration'
-require 'ddtrace/patcher'
-require 'ddtrace/augmentation'
-require 'ddtrace/metrics'
+# Load appsec
+require 'datadog/appsec/autoload' # TODO: datadog/appsec?
 
-# \Datadog global namespace that includes all tracing functionality for Tracer and Span classes.
-module Datadog
-  extend Augmentation
-  extend Configuration
-
-  # Load and extend Contrib by default
-  require 'ddtrace/contrib/extensions'
-  extend Contrib::Extensions
-
-  # Load and extend OpenTelemetry compatibility by default
-  require 'ddtrace/opentelemetry/extensions'
-  extend OpenTelemetry::Extensions
-
-  # Add shutdown hook:
-  # Ensures the tracer has an opportunity to flush traces
-  # and cleanup before terminating the process.
-  at_exit { Datadog.shutdown! }
-end
-
-require 'ddtrace/contrib/action_cable/integration'
-require 'ddtrace/contrib/action_pack/integration'
-require 'ddtrace/contrib/action_view/integration'
-require 'ddtrace/contrib/active_model_serializers/integration'
-require 'ddtrace/contrib/active_record/integration'
-require 'ddtrace/contrib/active_support/integration'
-require 'ddtrace/contrib/aws/integration'
-require 'ddtrace/contrib/concurrent_ruby/integration'
-require 'ddtrace/contrib/dalli/integration'
-require 'ddtrace/contrib/delayed_job/integration'
-require 'ddtrace/contrib/elasticsearch/integration'
-require 'ddtrace/contrib/ethon/integration'
-require 'ddtrace/contrib/excon/integration'
-require 'ddtrace/contrib/faraday/integration'
-require 'ddtrace/contrib/grape/integration'
-require 'ddtrace/contrib/graphql/integration'
-require 'ddtrace/contrib/grpc/integration'
-require 'ddtrace/contrib/http/integration'
-require 'ddtrace/contrib/integration'
-require 'ddtrace/contrib/presto/integration'
-require 'ddtrace/contrib/mysql2/integration'
-require 'ddtrace/contrib/mongodb/integration'
-require 'ddtrace/contrib/racecar/integration'
-require 'ddtrace/contrib/rack/integration'
-require 'ddtrace/contrib/rails/integration'
-require 'ddtrace/contrib/rake/integration'
-require 'ddtrace/contrib/redis/integration'
-require 'ddtrace/contrib/resque/integration'
-require 'ddtrace/contrib/rest_client/integration'
-require 'ddtrace/contrib/sequel/integration'
-require 'ddtrace/contrib/shoryuken/integration'
-require 'ddtrace/contrib/sidekiq/integration'
-require 'ddtrace/contrib/sinatra/integration'
-require 'ddtrace/contrib/sucker_punch/integration'
-require 'ddtrace/monkey'
+# Load other products (must follow tracing)
+require 'datadog/profiling'
+require 'datadog/ci'
+require 'datadog/kit'