datadog 2.41.0 → 2.42.0

data/lib/datadog/tracing/distributed/trace_state/datadog.rb

@@ -0,0 +1,233 @@
+# frozen_string_literal: true
+
+require_relative "ext"
+
+module Datadog
+  module Tracing
+    module Distributed
+      class TraceState
+        # Encodes and decodes Datadog propagation state.
+        class Datadog
+          class << self
+            def from_digest(digest)
+              ts_parent_id = if !digest.span_remote
+                format("%016x", digest.span_id || 0)
+              elsif digest.trace_distributed_tags&.key?(Tracing::Metadata::Ext::Distributed::TAG_DD_PARENT_ID)
+                digest.trace_distributed_tags[Tracing::Metadata::Ext::Distributed::TAG_DD_PARENT_ID]
+              end
+
+              new(
+                sampling_priority: digest.trace_sampling_priority,
+                origin: digest.trace_origin,
+                ts_parent_id: ts_parent_id,
+                tags: digest.trace_distributed_tags,
+                unknown_fields: digest.trace_state_unknown_fields,
+              )
+            end
+
+            def from_tracestate_member(value)
+              return new unless value
+
+              # @type var sampling_priority: ::Integer?
+              # @type var origin: ::String?
+              # @type var ts_parent_id: ::String?
+              # @type var tags: ::Hash[::String, ::String]?
+              # @type var unknown_fields: ::String?
+              sampling_priority = nil
+              origin = nil
+              ts_parent_id = nil
+              tags = nil
+              unknown_fields = nil
+
+              value.split(";").each do |pair|
+                key, field = pair.split(":", 2)
+                case key
+                when "s"
+                  sampling_priority = begin
+                    Integer(field)
+                  rescue
+                    nil
+                  end
+                when "o"
+                  origin = field
+                when "p"
+                  ts_parent_id = field
+                when /^t\./
+                  key = key.to_s
+                  field = field.to_s
+                  key.slice!(0..1)
+                  next if key == Tracing::Metadata::Ext::Distributed::TID
+
+                  field.tr!("~", "=")
+                  current_tags = tags || {}
+                  current_tags["#{Tracing::Metadata::Ext::Distributed::TAGS_PREFIX}#{key}"] = field
+                  tags = current_tags
+                else
+                  current_unknown_fields = unknown_fields || +""
+                  current_unknown_fields << pair << ";"
+                  unknown_fields = current_unknown_fields
+                end
+              end
+
+              new(
+                sampling_priority: sampling_priority,
+                origin: origin,
+                ts_parent_id: ts_parent_id,
+                tags: tags,
+                unknown_fields: unknown_fields,
+              )
+            end
+
+            # Serializes a {Hash<String,String>} into a `x-datadog-tags`-compatible
+            # String.
+            #
+            # @param tags [Hash<String,String>] trace tag hash
+            # @return [String] serialized tags hash
+            # @raise [EncodingError] if tags cannot be serialized to the `x-datadog-tags` format
+            def encode(tags)
+              tags.map do |raw_key, raw_value|
+                key = raw_key.to_s
+                value = raw_value.to_s
+
+                unless Ext::Datadog::VALID_KEY_CHARS.match?(key)
+                  raise EncodingError, "Invalid key `#{key}` for value `#{value}`"
+                end
+                unless Ext::Datadog::VALID_VALUE_CHARS.match?(value)
+                  raise EncodingError, "Invalid value `#{value}` for key `#{key}`"
+                end
+
+                "#{key}=#{value.strip}"
+              end.join(",")
+            rescue => e
+              raise EncodingError, "Error encoding tags `#{tags}`: `#{e.class}: #{e.message}`"
+            end
+
+            # Deserializes a `x-datadog-tags`-formatted String into a {Hash<String,String>}.
+            #
+            # @param string [String] tags as serialized by {#encode}
+            # @return [Hash<String,String>] decoded input as a hash of strings
+            # @raise [DecodingError] if string does not conform to the `x-datadog-tags` format
+            def decode(string)
+              result = (string.split(",").map do |raw_tag|
+                raw_tag.split("=", 2).tap do |raw_key, raw_value|
+                  key = raw_key.to_s
+                  value = raw_value.to_s
+
+                  raise DecodingError, "Invalid key: #{key}" unless Ext::Datadog::VALID_KEY_CHARS.match?(key)
+                  raise DecodingError, "Invalid value: #{value}" unless Ext::Datadog::VALID_VALUE_CHARS.match?(value)
+
+                  value.strip!
+                end
+              end).to_h
+
+              raise DecodingError, "Invalid empty tags: #{string}" if result.empty? && !string.empty?
+
+              result
+            end
+          end
+
+          # An error occurred during distributed tags encoding.
+          # See {#message} for more information.
+          class EncodingError < StandardError
+          end
+
+          # An error occurred during distributed tags decoding.
+          # See {#message} for more information.
+          class DecodingError < StandardError
+          end
+
+          attr_reader :sampling_priority, :origin, :ts_parent_id, :tags, :unknown_fields
+
+          def initialize(sampling_priority: nil, origin: nil, ts_parent_id: nil, tags: nil, unknown_fields: nil)
+            @sampling_priority = sampling_priority
+            @origin = origin
+            @ts_parent_id = ts_parent_id
+            @tags = tags
+            @unknown_fields = unknown_fields
+          end
+
+          # Builds the `dd=` tracestate member.
+          def to_s
+            vendor = +"dd="
+            if ts_parent_id
+              field = "p:#{ts_parent_id};"
+              vendor << field if vendor.bytesize + field.bytesize <= (Ext::TRACESTATE_VALUE_SIZE_LIMIT + 1)
+            end
+            if sampling_priority
+              field = "s:#{sampling_priority};"
+              vendor << field if vendor.bytesize + field.bytesize <= (Ext::TRACESTATE_VALUE_SIZE_LIMIT + 1)
+            end
+            if origin
+              field = "o:#{serialize_origin(origin)};"
+              vendor << field if vendor.bytesize + field.bytesize <= (Ext::TRACESTATE_VALUE_SIZE_LIMIT + 1)
+            end
+
+            # Replacing this with safe navigation causes a LocalJumpError on Rubies <= 3.0.
+            if tags # rubocop:disable Style/SafeNavigation
+              tags.each do |name, value|
+                tag_field = "t.#{serialize_tag_key(name)}:#{serialize_tag_value(value)};"
+                break if vendor.bytesize + tag_field.bytesize > (Ext::TRACESTATE_VALUE_SIZE_LIMIT + 1)
+
+                vendor << tag_field
+              end
+            end
+
+            if unknown_fields && vendor.bytesize + unknown_fields.bytesize <= (Ext::TRACESTATE_VALUE_SIZE_LIMIT + 1)
+              vendor << unknown_fields
+            end
+            return "" if vendor.bytesize == 3
+
+            vendor.chop!
+            vendor
+          end
+
+          private
+
+          # If any characters in <origin_value> are invalid, replace each invalid character with 0x5F (underscore).
+          # Invalid characters are: characters outside the ASCII range 0x20 to 0x7E,
+          # 0x2C (comma), 0x3B (semi-colon), and 0x7E (tilde).
+          # Then, remap 0x3D (equals) to 0x7E (tilde)
+          def serialize_origin(value)
+            # DEV: It's unlikely that characters will be out of range, as they mostly
+            # DEV: come from Datadog-controlled sources.
+            # DEV: Trying to `match?` is measurably faster than a `gsub` that does not match.
+            invalid_chars = Ext::Datadog::INVALID_ORIGIN_CHARS
+            value = invalid_chars.match?(value) ? value.gsub(invalid_chars, "_") : value
+            remapped_chars = Ext::Datadog::REMAP_ORIGIN_CHARS
+            remapped_chars.match?(value) ? value.gsub(remapped_chars, "~") : value
+          end
+
+          # Serialize `_dd.p.{key}` by first removing the `_dd.p.` prefix.
+          # Then replacing invalid characters with `_`.
+          #
+          # The argument `name` is always frozen.
+          # Returns a new String object for the serialized key.
+          def serialize_tag_key(name)
+            key = name.delete_prefix(Tracing::Metadata::Ext::Distributed::TAGS_PREFIX)
+
+            # DEV: It's unlikely that characters will be out of range, as they mostly
+            # DEV: come from Datadog-controlled sources.
+            # DEV: Trying to `match?` is measurably faster than a `gsub!` that does not match.
+            invalid_chars = Ext::Datadog::INVALID_TAG_KEY_CHARS
+            key.gsub!(invalid_chars, "_") if invalid_chars.match?(key)
+            key
+          end
+
+          # Replaces invalid characters with `_`, then replaces `=` with `~`.
+          #
+          # The argument `value` belongs to {TraceDigest}, thus should not be directly modified.
+          # Returns a new String object for the serialized value.
+          def serialize_tag_value(value)
+            # DEV: It's unlikely that characters will be out of range, as they mostly
+            # DEV: come from Datadog-controlled sources.
+            # DEV: Trying to `match?` is measurably faster than a `gsub` that does not match.
+            invalid_chars = Ext::Datadog::INVALID_TAG_VALUE_CHARS
+            serialized = invalid_chars.match?(value) ? value.gsub(invalid_chars, "_") : value
+            # DEV: Checking for an unlikely '=' is faster than a no-op `tr`.
+            serialized.include?("=") ? serialized.tr("=", "~") : serialized
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/distributed/trace_state/ext.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require "set"
+
+require_relative "../../sampling/ext"
+
+module Datadog
+  module Tracing
+    module Distributed
+      class TraceState
+        module Ext
+          TRACESTATE_MAX_SIZE_LIMIT = 512
+          TRACESTATE_MAX_LIST_VENDORS = 32
+          TRACESTATE_VALUE_SIZE_LIMIT = 256
+
+          module Datadog
+            # ASCII characters 32-126, except `,`, `=`, and ` `. At least one character.
+            VALID_KEY_CHARS = /\A(?:(?![,= ])[\u0020-\u007E])+\Z/.freeze
+            # ASCII characters 32-126, except `,`. At least one character.
+            VALID_VALUE_CHARS = /\A(?:(?!,)[\u0020-\u007E])+\Z/.freeze
+            INVALID_ORIGIN_CHARS = /[\u0000-\u0019,;~\u007F-\u{10FFFF}]/.freeze
+            REMAP_ORIGIN_CHARS = /=/.freeze
+            INVALID_TAG_KEY_CHARS = /[\u0000-\u0020,=\u007F-\u{10FFFF}]/.freeze
+            INVALID_TAG_VALUE_CHARS = /[\u0000-\u001F,;\u007E-\u{10FFFF}]/.freeze
+          end
+
+          module OpenTelemetry
+            MAX_THRESHOLD = 1 << 56
+            MAX_ENCODABLE_THRESHOLD = MAX_THRESHOLD - 1
+            UINT64_MODULO = 1 << 64
+            UINT64_MASK = UINT64_MODULO - 1
+            VALID_THRESHOLD = /\A[0-9a-f]{1,14}\z/
+            VALID_RANDOM_VALUE = /\A[0-9a-f]{14}\z/
+            NON_PROBABILITY_DECISIONS = Set[
+              Sampling::Ext::Decision::MANUAL,
+              Sampling::Ext::Decision::ASM,
+              Sampling::Ext::Decision::AI_GUARD,
+            ].freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/distributed/trace_state/open_telemetry.rb

@@ -0,0 +1,156 @@
+# frozen_string_literal: true
+
+require_relative "../../sampling/rate_sampler"
+require_relative "ext"
+
+module Datadog
+  module Tracing
+    module Distributed
+      class TraceState
+        # Encodes consistent probability sampling in the OpenTelemetry `ot=` tracestate member.
+        # @see https://opentelemetry.io/docs/specs/otel/trace/tracestate-probability-sampling/
+        class OpenTelemetry
+          class << self
+            def from_digest(digest, propagate_sampling: true)
+              new(
+                random_value: (digest.trace_otel_random_value if propagate_sampling),
+                threshold: (digest.trace_otel_threshold if propagate_sampling),
+                unknown_fields: digest.trace_otel_unknown_fields,
+              )
+            end
+
+            # Parses known fields and retains unknown fields for propagation.
+            def from_tracestate_member(value)
+              return new unless value
+
+              # @type var random_value: ::String?
+              # @type var threshold: ::String?
+              random_value = nil
+              threshold = nil
+              unknown = []
+
+              value.split(";").each do |pair|
+                key, field = pair.split(":", 2)
+                case key
+                when "rv"
+                  random_value = field
+                when "th"
+                  threshold = field
+                else
+                  unknown << pair
+                end
+              end
+
+              threshold = nil if threshold && !Ext::OpenTelemetry::VALID_THRESHOLD.match?(threshold)
+              random_value = nil if random_value && !Ext::OpenTelemetry::VALID_RANDOM_VALUE.match?(random_value)
+
+              new(
+                random_value: random_value,
+                threshold: threshold,
+                unknown_fields: unknown.empty? ? nil : "#{unknown.join(";")};",
+              )
+            end
+          end
+
+          attr_reader :random_value, :threshold, :unknown_fields
+
+          def initialize(random_value: nil, threshold: nil, unknown_fields: nil)
+            @random_value = random_value
+            @threshold = threshold
+            @unknown_fields = unknown_fields
+          end
+
+          # Resolves fields that represent the trace's current sampling decision.
+          def outbound(
+            trace_id:,
+            sampling_priority:,
+            decision_maker:,
+            applied_rate:,
+            rate_limiter_rate:,
+            distributed_sampling_priority:
+          )
+            if Ext::OpenTelemetry::NON_PROBABILITY_DECISIONS.include?(decision_maker)
+              return self.class.new(random_value: random_value, unknown_fields: unknown_fields)
+            end
+
+            trace_kept = sampling_priority && sampling_priority >= Sampling::Ext::Priority::AUTO_KEEP
+            if rate_limiter_rate && !trace_kept
+              return self.class.new(random_value: random_value, unknown_fields: unknown_fields)
+            end
+
+            return self if random_value || threshold
+            return self.class.new(unknown_fields: unknown_fields) if distributed_sampling_priority || !applied_rate
+
+            outbound_threshold = threshold_for(applied_rate)
+            outbound_random_value = reconcile_random_value(
+              random_value_for(trace_id),
+              outbound_threshold,
+              !!trace_kept
+            )
+            self.class.new(
+              random_value: format_random_value(outbound_random_value),
+              threshold: format_threshold(outbound_threshold),
+              unknown_fields: unknown_fields,
+            )
+          end
+
+          def to_s
+            fields = []
+            fields << "rv:#{random_value}" if random_value
+            fields << "th:#{threshold}" if threshold
+            fields.concat(unknown_fields.chomp(";").split(";")) if unknown_fields
+
+            value = +""
+            fields.each do |field|
+              separator = value.empty? ? "" : ";"
+              break if value.bytesize + separator.bytesize + field.bytesize > Ext::TRACESTATE_VALUE_SIZE_LIMIT
+
+              value << separator << field
+            end
+            value.empty? ? "" : "ot=#{value}"
+          end
+
+          private
+
+          # Converts Datadog's Knuth hash into OTel's 56-bit random value.
+          def random_value_for(trace_id)
+            hash = (trace_id * Sampling::RateSampler::KNUTH_FACTOR) % Ext::OpenTelemetry::UINT64_MODULO
+            (~hash & Ext::OpenTelemetry::UINT64_MASK) >> 8
+          end
+
+          # Converts a sample rate into a 56-bit rejection threshold.
+          def threshold_for(rate)
+            threshold = ((1.0 - rate) * Ext::OpenTelemetry::MAX_THRESHOLD).round
+            return 0 if threshold < 0
+            return Ext::OpenTelemetry::MAX_ENCODABLE_THRESHOLD if threshold > Ext::OpenTelemetry::MAX_ENCODABLE_THRESHOLD
+
+            threshold
+          end
+
+          # Preserve Datadog's 64-bit decision after expressing it with 56-bit OTel values.
+          def reconcile_random_value(random_value, threshold, kept)
+            if kept && random_value < threshold
+              threshold
+            elsif !kept && random_value >= threshold
+              [threshold - 1, 0].max
+            else
+              random_value
+            end
+          end
+
+          def format_random_value(random_value)
+            format("%014x", random_value)
+          end
+
+          # OTel thresholds omit trailing zero nibbles but retain leading zeroes.
+          def format_threshold(threshold)
+            return "0" if threshold.zero?
+
+            trailing_nibbles = ((threshold & -threshold).bit_length - 1) / 4
+            (threshold >> (trailing_nibbles * 4)).to_s(16).rjust(14 - trailing_nibbles, "0")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/distributed/trace_state.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+require_relative "../../core/utils"
+require_relative "trace_state/ext"
+require_relative "trace_state/datadog"
+require_relative "trace_state/open_telemetry"
+
+module Datadog
+  module Tracing
+    module Distributed
+      # Parsed vendor state from a W3C `tracestate` header.
+      # @api private
+      class TraceState
+        class << self
+          def from_digest(digest, propagate_sampling: true)
+            new(
+              unknown_vendors: digest.trace_state,
+              datadog: Datadog.from_digest(digest),
+              open_telemetry: OpenTelemetry.from_digest(digest, propagate_sampling: propagate_sampling),
+            )
+          end
+
+          def serialize_digest(digest)
+            serialize(
+              unknown_vendors: digest.trace_state,
+              datadog: Datadog.from_digest(digest),
+              open_telemetry: OpenTelemetry.from_digest(digest),
+            )
+          end
+
+          # Parses owned members while retaining other vendors unchanged.
+          def extract(tracestate)
+            vendors = split(tracestate) || []
+            datadog_index = vendors.index { |vendor| vendor.start_with?("dd=") }
+            datadog_value = vendors.delete_at(datadog_index).delete_prefix("dd=") if datadog_index
+            otel_index = vendors.index { |vendor| vendor.start_with?("ot=") }
+            otel_value = vendors.delete_at(otel_index).delete_prefix("ot=") if otel_index
+
+            new(
+              unknown_vendors: vendors.empty? ? nil : vendors.join(","),
+              datadog: Datadog.from_tracestate_member(datadog_value),
+              open_telemetry: OpenTelemetry.from_tracestate_member(otel_value),
+            )
+          end
+
+          # Partial members must not be propagated, but complete members within the limits are retained.
+          def split(tracestate)
+            return if tracestate.nil? || tracestate.empty?
+
+            remove_last_vendor = false
+            if tracestate.bytesize > Ext::TRACESTATE_MAX_SIZE_LIMIT
+              remove_last_vendor = tracestate.byteslice(Ext::TRACESTATE_MAX_SIZE_LIMIT, 1) != ","
+              tracestate = tracestate.byteslice(0, Ext::TRACESTATE_MAX_SIZE_LIMIT + 1) #: String
+              tracestate = tracestate.chop
+            end
+
+            tracestate = ::Datadog::Core::Utils.utf8_encode(tracestate, placeholder: nil)
+            return unless tracestate
+
+            vendors = tracestate.split(",", Ext::TRACESTATE_MAX_LIST_VENDORS + 1)
+            vendors.pop if vendors.length > Ext::TRACESTATE_MAX_LIST_VENDORS || remove_last_vendor
+            vendors.each(&:strip!)
+            vendors.pop while vendors.last == ""
+            vendors
+          end
+
+          private
+
+          def serialize(unknown_vendors:, datadog:, open_telemetry:)
+            leading_vendors = select_leading_vendors(datadog, open_telemetry)
+            vendors = split(unknown_vendors)
+
+            if leading_vendors.empty?
+              return unless vendors && !vendors.empty?
+
+              return vendors.join(",")
+            end
+
+            vendors&.reject! { |vendor| vendor.start_with?("dd=", "ot=") }
+
+            tracestate = leading_vendors.join(",")
+            if vendors && !vendors.empty?
+              vendors.first(Ext::TRACESTATE_MAX_LIST_VENDORS - leading_vendors.size).each do |vendor|
+                break if tracestate.bytesize + vendor.bytesize + 1 > Ext::TRACESTATE_MAX_SIZE_LIMIT
+
+                tracestate << "," << vendor
+              end
+            end
+
+            tracestate
+          end
+
+          def select_leading_vendors(datadog, open_telemetry)
+            leading_vendors = []
+            add_leading_vendor(leading_vendors, datadog)
+            add_leading_vendor(leading_vendors, open_telemetry)
+
+            leading_vendors
+          end
+
+          def add_leading_vendor(leading_vendors, state)
+            vendor = state.to_s
+            return if vendor.empty?
+
+            combined_size = leading_vendors.sum { |current| current.bytesize } +
+              leading_vendors.size + vendor.bytesize
+            leading_vendors << vendor if combined_size <= Ext::TRACESTATE_MAX_SIZE_LIMIT
+          end
+        end
+
+        attr_reader :unknown_vendors, :datadog, :open_telemetry
+
+        def initialize(unknown_vendors: nil, datadog: nil, open_telemetry: nil)
+          @unknown_vendors = unknown_vendors
+          @datadog = datadog || Datadog.new
+          @open_telemetry = open_telemetry || OpenTelemetry.new
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/otel_thread_context.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Tracing
+    # Publishes a per-thread OpenTelemetry context record into a thread-local slot,
+    # so an out-of-process reader (e.g. the eBPF profiler) can discover it.
+    #
+    # See the "OTel Thread Context" OTEP:
+    # https://github.com/open-telemetry/opentelemetry-specification/blob/main/oteps/profiles/4947-thread-ctx.md
+    #
+    # Native functions are implemented in ext/libdatadog_api/otel_thread_context.c
+    module OTelThreadContext
+      class << self
+        def supported?
+          Datadog::Core::LIBDATADOG_API_FAILURE.nil? && _native_supported?
+        end
+
+        def enable!
+          return false unless supported?
+
+          _native_enable
+        end
+
+        def set(trace_id:, span_id:, local_root_span_id:)
+          _native_set(trace_id, span_id, local_root_span_id)
+        end
+      end
+    end
+  end
+end