datadog 2.4.0 → 2.44.0

data/lib/datadog/tracing/trace_operation.rb

@@ -1,16 +1,16 @@
 # frozen_string_literal: true
 
-require_relative '../core/environment/identity'
-require_relative '../core/utils'
-require_relative 'tracer'
-require_relative 'event'
-require_relative 'metadata/tagging'
-require_relative 'sampling/ext'
-require_relative 'span_operation'
-require_relative 'trace_digest'
-require_relative 'correlation'
-require_relative 'trace_segment'
-require_relative 'utils'
+require_relative "../core/environment/identity"
+require_relative "../core/utils"
+require_relative "event"
+require_relative "metadata/tagging"
+require_relative "sampling/ext"
+require_relative "distributed/trace_state"
+require_relative "span_operation"
+require_relative "trace_digest"
+require_relative "correlation"
+require_relative "trace_segment"
+require_relative "utils"
 
 module Datadog
   module Tracing
@@ -28,6 +28,8 @@ module Datadog
       include Metadata::Tagging
 
       DEFAULT_MAX_LENGTH = 100_000
+      AUTO_SAMPLING_PRIORITIES = [Sampling::Ext::Priority::AUTO_KEEP, Sampling::Ext::Priority::AUTO_REJECT].freeze
+      RECONSIDERABLE_DECISIONS = [Sampling::Ext::Decision::DEFAULT, Sampling::Ext::Decision::AGENT_RATE].freeze
 
       attr_accessor \
         :agent_sample_rate,
@@ -37,24 +39,32 @@ module Datadog
         :rule_sample_rate,
         :sample_rate,
         :sampling_priority,
-        :remote_parent
+        :remote_parent,
+        :baggage
 
       attr_reader \
+        :logger,
         :active_span_count,
         :active_span,
         :id,
         :max_length,
         :parent_span_id,
-        :trace_state,
-        :trace_state_unknown_fields
+        :trace_state
 
       attr_writer \
         :name,
-        :resource,
         :sampled,
         :service
 
+      # Creates a new TraceOperation.
+      #
+      # @param span_links [Array<Datadog::Tracing::SpanLink>, nil] links to attach to the local root span.
+      # @param auto_finish [Boolean] when true, automatically finishes the trace when the local root span finishes.
+      #   When false, the trace remains unfinished until {#finish!} is called.
+      #   This is useful when this {TraceOperation} represents the continuation of a remote {TraceDigest},
+      #   in which case local root spans in this {TraceOperation} are children of the {TraceDigest}'s last active span.
       def initialize(
+        logger: Datadog.logger,
         agent_sample_rate: nil,
         events: nil,
         hostname: nil,
@@ -63,6 +73,7 @@ module Datadog
         name: nil,
         origin: nil,
         parent_span_id: nil,
+        span_links: nil,
         rate_limiter_rate: nil,
         resource: nil,
         rule_sample_rate: nil,
@@ -71,21 +82,26 @@ module Datadog
         sampling_priority: nil,
         service: nil,
         profiling_enabled: nil,
+        apm_tracing_enabled: nil,
         tags: nil,
         metrics: nil,
         trace_state: nil,
-        trace_state_unknown_fields: nil,
         remote_parent: false,
-        tracer: nil
-
+        distributed_sampling_priority: false,
+        tracer: nil, # DEV-3.0: deprecated, remove in 3.0
+        baggage: nil,
+        auto_finish: true
       )
+        @logger = logger
+
         # Attributes
         @id = id || Tracing::Utils::TraceId.next_id
         @max_length = max_length || DEFAULT_MAX_LENGTH
         @parent_span_id = parent_span_id
-        @sampled = sampled.nil? ? true : sampled
+        @sampled = sampled.nil? || sampled
         @remote_parent = remote_parent
-
+        @distributed_sampling_priority = distributed_sampling_priority
+        @span_links = span_links
         # Tags
         @agent_sample_rate = agent_sample_rate
         @hostname = hostname
@@ -98,9 +114,9 @@ module Datadog
         @sampling_priority = sampling_priority
         @service = service
         @profiling_enabled = profiling_enabled
-        @trace_state = trace_state
-        @trace_state_unknown_fields = trace_state_unknown_fields
-        @tracer = tracer
+        @apm_tracing_enabled = apm_tracing_enabled
+        @trace_state = trace_state || Distributed::TraceState.new
+        @baggage = baggage
 
         # Generic tags
         set_tags(tags) if tags
@@ -113,6 +129,9 @@ module Datadog
         @events = events || Events.new
         @finished = false
         @spans = []
+        @auto_finish = !!auto_finish
+        @flushed = false
+        @propagated = false
       end
 
       def full?
@@ -156,21 +175,38 @@ module Datadog
         set_tag(Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER, Tracing::Sampling::Ext::Decision::MANUAL)
       end
 
+      def resource=(value)
+        previous_resource = @resource
+        @resource = value
+
+        return if !!previous_resource || value.nil?
+
+        events.trace_resource_change.publish(self)
+      rescue => e
+        logger.debug { "Error updating trace resource: #{e.class}: #{e.message} Backtrace: #{e.backtrace.first(3)}" }
+      end
+
       def name
-        @name || (root_span && root_span.name)
+        @name || root_span&.name
       end
 
       def resource
-        @resource || (root_span && root_span.resource)
+        @resource || root_span&.resource
       end
 
       # When retrieving tags or metrics we need to include root span tags for sampling purposes
       def get_tag(key)
-        super || (root_span && root_span.get_tag(key))
+        super || root_span&.get_tag(key)
       end
 
       def get_metric(key)
-        super || (root_span && root_span.get_metric(key))
+        super || root_span&.get_metric(key)
+      end
+
+      def set_distributed_source(product_bit)
+        source = get_tag(Metadata::Ext::Distributed::TAG_TRACE_SOURCE)&.to_i(16) || 0
+        source |= product_bit
+        set_tag(Metadata::Ext::Distributed::TAG_TRACE_SOURCE, format("%02X", source))
       end
 
       def tags
@@ -188,12 +224,19 @@ module Datadog
         !@resource.nil?
       end
 
+      def reconsider_resource_sample?
+        return false if @resource.nil?
+
+        reconsider_rule_sample?
+      end
+
       def service
-        @service || (root_span && root_span.service)
+        @service || root_span&.service
       end
 
       def measure(
         op_name,
+        logger: Datadog.logger,
         events: nil,
         on_error: nil,
         resource: nil,
@@ -207,7 +250,11 @@ module Datadog
         # Don't allow more span measurements if the
         # trace is already completed. Prevents multiple
         # root spans with parent_span_id = 0.
-        return yield(SpanOperation.new(op_name), TraceOperation.new) if finished? || full?
+        if finished? || full?
+          return yield(
+            SpanOperation.new(op_name, logger: logger),
+            TraceOperation.new(logger: logger))
+        end
 
         # Create new span
         span_op = build_span(
@@ -228,6 +275,7 @@ module Datadog
 
       def build_span(
         op_name,
+        logger: Datadog.logger,
         events: nil,
         on_error: nil,
         resource: nil,
@@ -237,50 +285,55 @@ module Datadog
         type: nil,
         id: nil
       )
-        begin
-          # Resolve span options:
-          # Parent, service name, etc.
-          # Add default options
-          trace_id = @id
-          parent = @active_span
-
-          # Use active span's span ID if available. Otherwise, the parent span ID.
-          # Necessary when this trace continues from another, e.g. distributed trace.
-          parent_id = parent ? parent.id : @parent_span_id || 0
-
-          # Build events
-          events ||= SpanOperation::Events.new
-
-          # Before start: activate the span, publish events.
-          events.before_start.subscribe do |span_op|
-            start_span(span_op)
-          end
+        # Resolve span options:
+        # Parent, service name, etc.
+        # Add default options
+        trace_id = @id
+        parent = @active_span
 
-          # After finish: deactivate the span, record, publish events.
-          events.after_finish.subscribe do |span, span_op|
-            finish_span(span, span_op, parent)
-          end
+        # Use active span's span ID if available. Otherwise, the parent span ID.
+        # Necessary when this trace continues from another, e.g. distributed trace.
+        parent_id = parent ? parent.id : @parent_span_id || 0
+
+        # Build events
+        span_events = events || SpanOperation::Events.new(logger: logger)
 
-          # Build a new span operation
-          SpanOperation.new(
-            op_name,
-            events: events,
-            on_error: on_error,
-            parent_id: parent_id,
-            resource: resource || op_name,
-            service: service,
-            start_time: start_time,
-            tags: tags,
-            trace_id: trace_id,
-            type: type,
-            id: id
-          )
-        rescue StandardError => e
-          Datadog.logger.debug { "Failed to build new span: #{e}" }
-
-          # Return dummy span
-          SpanOperation.new(op_name)
+        # Before start: activate the span, publish events.
+        span_events.before_start.subscribe do |span_op|
+          start_span(span_op)
         end
+
+        # Before finish: allow enrichment before the span is finalized, publish events.
+        span_events.before_finish.subscribe do |span_op|
+          before_finish_span(span_op)
+        end
+
+        # After finish: deactivate the span, record, publish events.
+        span_events.after_finish.subscribe do |span, span_op|
+          finish_span(span, span_op, parent)
+        end
+
+        # Build a new span operation
+        SpanOperation.new(
+          op_name,
+          logger: logger,
+          events: span_events,
+          on_error: on_error,
+          parent_id: parent_id,
+          links: (@root_span.nil? ? @span_links : nil),
+          resource: resource || op_name,
+          service: service,
+          start_time: start_time,
+          tags: tags,
+          trace_id: trace_id,
+          type: type,
+          id: id
+        )
+      rescue => e
+        logger.debug { "Failed to build new span: #{e.class}: #{e.message}" }
+
+        # Return dummy span
+        SpanOperation.new(op_name, logger: logger)
       end
 
       # Returns a {TraceSegment} with all finished spans that can be flushed
@@ -294,6 +347,7 @@ module Datadog
         # Copy out completed spans
         spans = @spans.dup
         @spans = []
+        @flushed = true
 
         spans = yield(spans) if block_given?
 
@@ -301,6 +355,29 @@ module Datadog
         build_trace(spans, !finished)
       end
 
+      # When automatic context management is disabled (@auto_finish is false),
+      # this method finishes the trace, marking it as completed.
+      #
+      # The trace will **not** automatically finish when its local root span
+      # when @auto_finish is false, thus calling this method is mandatory
+      # in such scenario.
+      #
+      # Unfinished spans are discarded.
+      #
+      # This method is idempotent and safe to call after the trace is finished.
+      # It is also a no-op when @auto_finish is true, to prevent misuse.
+      #
+      # @!visibility private
+      def finish!
+        return if @auto_finish || finished?
+
+        @finished = true
+        @active_span = nil
+        @active_span_count = 0
+
+        events.trace_finished.publish(self)
+      end
+
       # Returns a set of trace headers used for continuing traces.
       # Used for propagation across execution contexts.
       # Data should reflect the active state of the trace.
@@ -308,17 +385,27 @@ module Datadog
       # We should move the sample call to inject and right before moving to new contexts(threads, forking etc.)
       def to_digest
         # Resolve current span ID
-        span_id = @active_span && @active_span.id
+        span_id = @active_span&.id
         span_id ||= @parent_span_id unless finished?
         # sample the trace_operation with the tracer
-        @tracer&.sample_trace(self) unless sampling_priority
+        @propagated = true
+        events.trace_propagated.publish(self)
+
+        open_telemetry = @trace_state.open_telemetry.outbound(
+          trace_id: @id,
+          sampling_priority: @sampling_priority,
+          decision_maker: get_tag(Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER),
+          applied_rate: @rule_sample_rate || @agent_sample_rate,
+          rate_limiter_rate: @rate_limiter_rate,
+          distributed_sampling_priority: @distributed_sampling_priority,
+        )
 
         TraceDigest.new(
           span_id: span_id,
-          span_name: (@active_span && @active_span.name),
-          span_resource: (@active_span && @active_span.resource),
-          span_service: (@active_span && @active_span.service),
-          span_type: (@active_span && @active_span.type),
+          span_name: @active_span && @active_span.name,
+          span_resource: @active_span && @active_span.resource,
+          span_service: @active_span && @active_span.service,
+          span_type: @active_span && @active_span.type,
           trace_distributed_tags: distributed_tags,
           trace_hostname: @hostname,
           trace_id: @id,
@@ -329,15 +416,19 @@ module Datadog
           trace_runtime_id: Core::Environment::Identity.id,
           trace_sampling_priority: @sampling_priority,
           trace_service: service,
-          trace_state: @trace_state,
-          trace_state_unknown_fields: @trace_state_unknown_fields,
-          span_remote: (@remote_parent && @active_span.nil?),
+          trace_state: @trace_state.unknown_vendors,
+          trace_state_unknown_fields: @trace_state.datadog.unknown_fields,
+          trace_otel_random_value: open_telemetry.random_value,
+          trace_otel_threshold: open_telemetry.threshold,
+          trace_otel_unknown_fields: open_telemetry.unknown_fields,
+          span_remote: @remote_parent && @active_span.nil?,
+          baggage: (@baggage.nil? || @baggage.empty?) ? nil : @baggage
         ).freeze
       end
 
       def to_correlation
         # Resolve current span ID
-        span_id = @active_span && @active_span.id
+        span_id = @active_span&.id
         span_id ||= @parent_span_id unless finished?
 
         Correlation::Identifier.new(
@@ -351,25 +442,25 @@ module Datadog
       def fork_clone
         self.class.new(
           agent_sample_rate: @agent_sample_rate,
-          events: (@events && @events.dup),
-          hostname: (@hostname && @hostname.dup),
+          events: @events&.dup,
+          hostname: @hostname&.dup,
           id: @id,
           max_length: @max_length,
-          name: (name && name.dup),
-          origin: (@origin && @origin.dup),
-          parent_span_id: (@active_span && @active_span.id) || @parent_span_id,
+          name: name&.dup,
+          origin: @origin&.dup,
+          parent_span_id: @active_span&.id || @parent_span_id,
           rate_limiter_rate: @rate_limiter_rate,
-          resource: (resource && resource.dup),
+          resource: resource&.dup,
           rule_sample_rate: @rule_sample_rate,
           sample_rate: @sample_rate,
           sampled: @sampled,
           sampling_priority: @sampling_priority,
-          service: (service && service.dup),
-          trace_state: (@trace_state && @trace_state.dup),
-          trace_state_unknown_fields: (@trace_state_unknown_fields && @trace_state_unknown_fields.dup),
+          service: service&.dup,
+          trace_state: @trace_state,
           tags: meta.dup,
           metrics: metrics.dup,
-          remote_parent: @remote_parent
+          remote_parent: @remote_parent,
+          distributed_sampling_priority: @distributed_sampling_priority
         )
       end
 
@@ -378,14 +469,28 @@ module Datadog
         include Tracing::Events
 
         attr_reader \
+          :span_before_finish,
           :span_before_start,
           :span_finished,
-          :trace_finished
+          :trace_finished,
+          :trace_propagated,
+          :trace_resource_change
 
         def initialize
+          @span_before_finish = SpanBeforeFinish.new
           @span_before_start = SpanBeforeStart.new
           @span_finished = SpanFinished.new
           @trace_finished = TraceFinished.new
+          @trace_propagated = TracePropagated.new
+          @trace_resource_change = TraceResourceChange.new
+        end
+
+        # Triggered just before a span is finalized, mirroring SpanOperation::Events::BeforeFinish.
+        # Subscribers can still mutate tags on the SpanOperation at this point.
+        class SpanBeforeFinish < Tracing::Event
+          def initialize
+            super(:span_before_finish)
+          end
         end
 
         # Triggered before a span starts.
@@ -402,6 +507,13 @@ module Datadog
           end
         end
 
+        #  Triggered when trace is being propagated between applications or contexts
+        class TracePropagated < Tracing::Event
+          def initialize
+            super(:trace_propagated)
+          end
+        end
+
         # Triggered when the trace finishes, regardless of error.
         class TraceFinished < Tracing::Event
           def initialize
@@ -418,6 +530,14 @@ module Datadog
             subscribe(&block)
           end
         end
+
+        # Triggered when the resource name is set
+        # This is used to reconsider trace sampling on resource name change
+        class TraceResourceChange < Tracing::Event
+          def initialize
+            super(:trace_resource_change)
+          end
+        end
       end
 
       private
@@ -433,7 +553,7 @@ module Datadog
 
         @active_span = span_op
 
-        set_root_span!(span_op) unless root_span
+        set_local_root_span!(span_op)
       end
 
       def deactivate_span!(span_op)
@@ -445,45 +565,55 @@ module Datadog
       end
 
       def start_span(span_op)
-        begin
-          activate_span!(span_op)
+        activate_span!(span_op)
 
-          # Update active span count
-          @active_span_count += 1
+        # Update active span count
+        @active_span_count += 1
 
-          # Publish :span_before_start event
-          events.span_before_start.publish(span_op, self)
-        rescue StandardError => e
-          Datadog.logger.debug { "Error starting span on trace: #{e} Backtrace: #{e.backtrace.first(3)}" }
-        end
+        # Publish :span_before_start event
+        events.span_before_start.publish(span_op, self)
+      rescue => e
+        logger.debug { "Error starting span on trace: #{e.class}: #{e.message} Backtrace: #{e.backtrace.first(3)}" }
       end
 
+      def before_finish_span(span_op)
+        # Publish :span_before_finish event
+        events.span_before_finish.publish(span_op, self)
+      rescue => e
+        logger.debug { "Error in before_finish_span on trace: #{e.class}: #{e.message} Backtrace: #{e.backtrace.first(3)}" }
+      end
+
+      # For traces with automatic context management (auto_finish),
+      # when the local root span finishes, the trace also finishes.
+      # The trace cannot receive new spans after finished.
+      #
+      # Without auto_finish, the trace can still receive spans
+      # until explicitly finished.
       def finish_span(span, span_op, parent)
-        begin
-          # Save finished span & root span
-          @spans << span unless span.nil?
+        # Save finished span & root span
+        @spans << span unless span.nil?
 
-          # Deactivate the span, re-activate parent.
-          deactivate_span!(span_op)
+        # Deactivate the span, re-activate parent.
+        deactivate_span!(span_op)
 
-          # Set finished, to signal root span has completed.
-          @finished = true if span_op == root_span
+        # Finish if the local root span is finished and automatic
+        # context management is enabled.
+        @finished = true if span_op == root_span && @auto_finish
 
-          # Update active span count
-          @active_span_count -= 1
+        # Update active span count
+        @active_span_count -= 1
 
-          # Publish :span_finished event
-          events.span_finished.publish(span, self)
+        # Publish :span_finished event
+        events.span_finished.publish(span, self)
 
-          # Publish :trace_finished event
-          events.trace_finished.publish(self) if finished?
-        rescue StandardError => e
-          Datadog.logger.debug { "Error finishing span on trace: #{e} Backtrace: #{e.backtrace.first(3)}" }
-        end
+        # Publish :trace_finished event
+        events.trace_finished.publish(self) if finished?
+      rescue => e
+        logger.debug { "Error finishing span on trace: #{e.class}: #{e.message} Backtrace: #{e.backtrace.first(3)}" }
       end
 
-      # Track the root span
-      def set_root_span!(span)
+      # Track the root {SpanOperation} object from the current execution context.
+      def set_local_root_span!(span)
         return if span.nil? || root_span
 
         @root_span = span
@@ -508,8 +638,9 @@ module Datadog
           service: service,
           tags: meta,
           metrics: metrics,
-          root_span_id: !partial ? root_span && root_span.id : nil,
+          root_span_id: (!partial) ? root_span&.id : nil,
           profiling_enabled: @profiling_enabled,
+          apm_tracing_enabled: @apm_tracing_enabled
         )
       end
 
@@ -520,12 +651,24 @@ module Datadog
         meta.select { |name, _| name.start_with?(Metadata::Ext::Distributed::TAGS_PREFIX) }
       end
 
+      def reconsider_rule_sample?
+        decision = get_tag(Metadata::Ext::Distributed::TAG_DECISION_MAKER)
+
+        return false if remote_parent || @propagated || @flushed
+        return false unless AUTO_SAMPLING_PRIORITIES.include?(@sampling_priority)
+        return false if decision && !RECONSIDERABLE_DECISIONS.include?(decision)
+
+        true
+      end
+
       def reset
         @root_span = nil
         @active_span = nil
         @active_span_count = 0
         @finished = false
         @spans = []
+        @flushed = false
+        @propagated = false
       end
     end
   end