datadog 2.17.0 → 2.37.0

data/lib/datadog/tracing/trace_operation.rb

@@ -27,6 +27,8 @@ module Datadog
       include Metadata::Tagging
 
       DEFAULT_MAX_LENGTH = 100_000
+      AUTO_SAMPLING_PRIORITIES = [Sampling::Ext::Priority::AUTO_KEEP, Sampling::Ext::Priority::AUTO_REJECT].freeze
+      RECONSIDERABLE_DECISIONS = [Sampling::Ext::Decision::DEFAULT, Sampling::Ext::Decision::AGENT_RATE].freeze
 
       attr_accessor \
         :agent_sample_rate,
@@ -51,10 +53,16 @@ module Datadog
 
       attr_writer \
         :name,
-        :resource,
         :sampled,
         :service
 
+      # Creates a new TraceOperation.
+      #
+      # @param span_links [Array<Datadog::Tracing::SpanLink>, nil] links to attach to the local root span.
+      # @param auto_finish [Boolean] when true, automatically finishes the trace when the local root span finishes.
+      #   When false, the trace remains unfinished until {#finish!} is called.
+      #   This is useful when this {TraceOperation} represents the continuation of a remote {TraceDigest},
+      #   in which case local root spans in this {TraceOperation} are children of the {TraceDigest}'s last active span.
       def initialize(
         logger: Datadog.logger,
         agent_sample_rate: nil,
@@ -65,6 +73,7 @@ module Datadog
         name: nil,
         origin: nil,
         parent_span_id: nil,
+        span_links: nil,
         rate_limiter_rate: nil,
         resource: nil,
         rule_sample_rate: nil,
@@ -79,8 +88,9 @@ module Datadog
         trace_state: nil,
         trace_state_unknown_fields: nil,
         remote_parent: false,
-        tracer: nil,
-        baggage: nil
+        tracer: nil, # DEV-3.0: deprecated, remove in 3.0
+        baggage: nil,
+        auto_finish: true
       )
         @logger = logger
 
@@ -90,7 +100,7 @@ module Datadog
         @parent_span_id = parent_span_id
         @sampled = sampled.nil? || sampled
         @remote_parent = remote_parent
-
+        @span_links = span_links
         # Tags
         @agent_sample_rate = agent_sample_rate
         @hostname = hostname
@@ -106,7 +116,6 @@ module Datadog
         @apm_tracing_enabled = apm_tracing_enabled
         @trace_state = trace_state
         @trace_state_unknown_fields = trace_state_unknown_fields
-        @tracer = tracer
         @baggage = baggage
 
         # Generic tags
@@ -120,6 +129,9 @@ module Datadog
         @events = events || Events.new
         @finished = false
         @spans = []
+        @auto_finish = !!auto_finish
+        @flushed = false
+        @propagated = false
       end
 
       def full?
@@ -163,21 +175,32 @@ module Datadog
         set_tag(Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER, Tracing::Sampling::Ext::Decision::MANUAL)
       end
 
+      def resource=(value)
+        previous_resource = @resource
+        @resource = value
+
+        return if !!previous_resource || value.nil?
+
+        events.trace_resource_change.publish(self)
+      rescue => e
+        logger.debug { "Error updating trace resource: #{e.class}: #{e.message} Backtrace: #{e.backtrace.first(3)}" }
+      end
+
       def name
-        @name || (root_span && root_span.name)
+        @name || root_span&.name
       end
 
       def resource
-        @resource || (root_span && root_span.resource)
+        @resource || root_span&.resource
       end
 
       # When retrieving tags or metrics we need to include root span tags for sampling purposes
       def get_tag(key)
-        super || (root_span && root_span.get_tag(key))
+        super || root_span&.get_tag(key)
       end
 
       def get_metric(key)
-        super || (root_span && root_span.get_metric(key))
+        super || root_span&.get_metric(key)
       end
 
       def set_distributed_source(product_bit)
@@ -201,8 +224,14 @@ module Datadog
         !@resource.nil?
       end
 
+      def reconsider_resource_sample?
+        return false if @resource.nil?
+
+        reconsider_rule_sample?
+      end
+
       def service
-        @service || (root_span && root_span.service)
+        @service || root_span&.service
       end
 
       def measure(
@@ -221,9 +250,11 @@ module Datadog
         # Don't allow more span measurements if the
         # trace is already completed. Prevents multiple
         # root spans with parent_span_id = 0.
-        return yield( # rubocop:disable Style/MultilineIfModifier
-          SpanOperation.new(op_name, logger: logger),
-          TraceOperation.new(logger: logger)) if finished? || full?
+        if finished? || full?
+          return yield(
+            SpanOperation.new(op_name, logger: logger),
+            TraceOperation.new(logger: logger))
+        end
 
         # Create new span
         span_op = build_span(
@@ -254,51 +285,55 @@ module Datadog
         type: nil,
         id: nil
       )
-        begin
-          # Resolve span options:
-          # Parent, service name, etc.
-          # Add default options
-          trace_id = @id
-          parent = @active_span
-
-          # Use active span's span ID if available. Otherwise, the parent span ID.
-          # Necessary when this trace continues from another, e.g. distributed trace.
-          parent_id = parent ? parent.id : @parent_span_id || 0
-
-          # Build events
-          events ||= SpanOperation::Events.new(logger: logger)
-
-          # Before start: activate the span, publish events.
-          events.before_start.subscribe do |span_op|
-            start_span(span_op)
-          end
+        # Resolve span options:
+        # Parent, service name, etc.
+        # Add default options
+        trace_id = @id
+        parent = @active_span
 
-          # After finish: deactivate the span, record, publish events.
-          events.after_finish.subscribe do |span, span_op|
-            finish_span(span, span_op, parent)
-          end
+        # Use active span's span ID if available. Otherwise, the parent span ID.
+        # Necessary when this trace continues from another, e.g. distributed trace.
+        parent_id = parent ? parent.id : @parent_span_id || 0
+
+        # Build events
+        span_events = events || SpanOperation::Events.new(logger: logger)
+
+        # Before start: activate the span, publish events.
+        span_events.before_start.subscribe do |span_op|
+          start_span(span_op)
+        end
+
+        # Before finish: allow enrichment before the span is finalized, publish events.
+        span_events.before_finish.subscribe do |span_op|
+          before_finish_span(span_op)
+        end
 
-          # Build a new span operation
-          SpanOperation.new(
-            op_name,
-            logger: logger,
-            events: events,
-            on_error: on_error,
-            parent_id: parent_id,
-            resource: resource || op_name,
-            service: service,
-            start_time: start_time,
-            tags: tags,
-            trace_id: trace_id,
-            type: type,
-            id: id
-          )
-        rescue StandardError => e
-          logger.debug { "Failed to build new span: #{e}" }
-
-          # Return dummy span
-          SpanOperation.new(op_name, logger: logger)
+        # After finish: deactivate the span, record, publish events.
+        span_events.after_finish.subscribe do |span, span_op|
+          finish_span(span, span_op, parent)
         end
+
+        # Build a new span operation
+        SpanOperation.new(
+          op_name,
+          logger: logger,
+          events: span_events,
+          on_error: on_error,
+          parent_id: parent_id,
+          links: (@root_span.nil? ? @span_links : nil),
+          resource: resource || op_name,
+          service: service,
+          start_time: start_time,
+          tags: tags,
+          trace_id: trace_id,
+          type: type,
+          id: id
+        )
+      rescue => e
+        logger.debug { "Failed to build new span: #{e.class}: #{e.message}" }
+
+        # Return dummy span
+        SpanOperation.new(op_name, logger: logger)
       end
 
       # Returns a {TraceSegment} with all finished spans that can be flushed
@@ -312,6 +347,7 @@ module Datadog
         # Copy out completed spans
         spans = @spans.dup
         @spans = []
+        @flushed = true
 
         spans = yield(spans) if block_given?
 
@@ -319,6 +355,29 @@ module Datadog
         build_trace(spans, !finished)
       end
 
+      # When automatic context management is disabled (@auto_finish is false),
+      # this method finishes the trace, marking it as completed.
+      #
+      # The trace will **not** automatically finish when its local root span
+      # when @auto_finish is false, thus calling this method is mandatory
+      # in such scenario.
+      #
+      # Unfinished spans are discarded.
+      #
+      # This method is idempotent and safe to call after the trace is finished.
+      # It is also a no-op when @auto_finish is true, to prevent misuse.
+      #
+      # @!visibility private
+      def finish!
+        return if @auto_finish || finished?
+
+        @finished = true
+        @active_span = nil
+        @active_span_count = 0
+
+        events.trace_finished.publish(self)
+      end
+
       # Returns a set of trace headers used for continuing traces.
       # Used for propagation across execution contexts.
       # Data should reflect the active state of the trace.
@@ -326,10 +385,11 @@ module Datadog
       # We should move the sample call to inject and right before moving to new contexts(threads, forking etc.)
       def to_digest
         # Resolve current span ID
-        span_id = @active_span && @active_span.id
+        span_id = @active_span&.id
         span_id ||= @parent_span_id unless finished?
         # sample the trace_operation with the tracer
-        @tracer&.sample_trace(self) unless sampling_priority
+        @propagated = true
+        events.trace_propagated.publish(self)
 
         TraceDigest.new(
           span_id: span_id,
@@ -350,13 +410,13 @@ module Datadog
           trace_state: @trace_state,
           trace_state_unknown_fields: @trace_state_unknown_fields,
           span_remote: @remote_parent && @active_span.nil?,
-          baggage: @baggage.nil? || @baggage.empty? ? nil : @baggage
+          baggage: (@baggage.nil? || @baggage.empty?) ? nil : @baggage
         ).freeze
       end
 
       def to_correlation
         # Resolve current span ID
-        span_id = @active_span && @active_span.id
+        span_id = @active_span&.id
         span_id ||= @parent_span_id unless finished?
 
         Correlation::Identifier.new(
@@ -370,22 +430,22 @@ module Datadog
       def fork_clone
         self.class.new(
           agent_sample_rate: @agent_sample_rate,
-          events: @events && @events.dup,
-          hostname: @hostname && @hostname.dup,
+          events: @events&.dup,
+          hostname: @hostname&.dup,
           id: @id,
           max_length: @max_length,
-          name: name && name.dup,
-          origin: @origin && @origin.dup,
-          parent_span_id: (@active_span && @active_span.id) || @parent_span_id,
+          name: name&.dup,
+          origin: @origin&.dup,
+          parent_span_id: @active_span&.id || @parent_span_id,
           rate_limiter_rate: @rate_limiter_rate,
-          resource: resource && resource.dup,
+          resource: resource&.dup,
           rule_sample_rate: @rule_sample_rate,
           sample_rate: @sample_rate,
           sampled: @sampled,
           sampling_priority: @sampling_priority,
-          service: service && service.dup,
-          trace_state: @trace_state && @trace_state.dup,
-          trace_state_unknown_fields: @trace_state_unknown_fields && @trace_state_unknown_fields.dup,
+          service: service&.dup,
+          trace_state: @trace_state&.dup,
+          trace_state_unknown_fields: @trace_state_unknown_fields&.dup,
           tags: meta.dup,
           metrics: metrics.dup,
           remote_parent: @remote_parent
@@ -397,14 +457,28 @@ module Datadog
         include Tracing::Events
 
         attr_reader \
+          :span_before_finish,
           :span_before_start,
           :span_finished,
-          :trace_finished
+          :trace_finished,
+          :trace_propagated,
+          :trace_resource_change
 
         def initialize
+          @span_before_finish = SpanBeforeFinish.new
           @span_before_start = SpanBeforeStart.new
           @span_finished = SpanFinished.new
           @trace_finished = TraceFinished.new
+          @trace_propagated = TracePropagated.new
+          @trace_resource_change = TraceResourceChange.new
+        end
+
+        # Triggered just before a span is finalized, mirroring SpanOperation::Events::BeforeFinish.
+        # Subscribers can still mutate tags on the SpanOperation at this point.
+        class SpanBeforeFinish < Tracing::Event
+          def initialize
+            super(:span_before_finish)
+          end
         end
 
         # Triggered before a span starts.
@@ -421,6 +495,13 @@ module Datadog
           end
         end
 
+        #  Triggered when trace is being propagated between applications or contexts
+        class TracePropagated < Tracing::Event
+          def initialize
+            super(:trace_propagated)
+          end
+        end
+
         # Triggered when the trace finishes, regardless of error.
         class TraceFinished < Tracing::Event
           def initialize
@@ -437,6 +518,14 @@ module Datadog
             subscribe(&block)
           end
         end
+
+        # Triggered when the resource name is set
+        # This is used to reconsider trace sampling on resource name change
+        class TraceResourceChange < Tracing::Event
+          def initialize
+            super(:trace_resource_change)
+          end
+        end
       end
 
       private
@@ -452,7 +541,7 @@ module Datadog
 
         @active_span = span_op
 
-        set_root_span!(span_op) unless root_span
+        set_local_root_span!(span_op)
       end
 
       def deactivate_span!(span_op)
@@ -464,45 +553,55 @@ module Datadog
       end
 
       def start_span(span_op)
-        begin
-          activate_span!(span_op)
+        activate_span!(span_op)
 
-          # Update active span count
-          @active_span_count += 1
+        # Update active span count
+        @active_span_count += 1
 
-          # Publish :span_before_start event
-          events.span_before_start.publish(span_op, self)
-        rescue StandardError => e
-          logger.debug { "Error starting span on trace: #{e} Backtrace: #{e.backtrace.first(3)}" }
-        end
+        # Publish :span_before_start event
+        events.span_before_start.publish(span_op, self)
+      rescue => e
+        logger.debug { "Error starting span on trace: #{e.class}: #{e.message} Backtrace: #{e.backtrace.first(3)}" }
+      end
+
+      def before_finish_span(span_op)
+        # Publish :span_before_finish event
+        events.span_before_finish.publish(span_op, self)
+      rescue => e
+        logger.debug { "Error in before_finish_span on trace: #{e.class}: #{e.message} Backtrace: #{e.backtrace.first(3)}" }
       end
 
+      # For traces with automatic context management (auto_finish),
+      # when the local root span finishes, the trace also finishes.
+      # The trace cannot receive new spans after finished.
+      #
+      # Without auto_finish, the trace can still receive spans
+      # until explicitly finished.
       def finish_span(span, span_op, parent)
-        begin
-          # Save finished span & root span
-          @spans << span unless span.nil?
+        # Save finished span & root span
+        @spans << span unless span.nil?
 
-          # Deactivate the span, re-activate parent.
-          deactivate_span!(span_op)
+        # Deactivate the span, re-activate parent.
+        deactivate_span!(span_op)
 
-          # Set finished, to signal root span has completed.
-          @finished = true if span_op == root_span
+        # Finish if the local root span is finished and automatic
+        # context management is enabled.
+        @finished = true if span_op == root_span && @auto_finish
 
-          # Update active span count
-          @active_span_count -= 1
+        # Update active span count
+        @active_span_count -= 1
 
-          # Publish :span_finished event
-          events.span_finished.publish(span, self)
+        # Publish :span_finished event
+        events.span_finished.publish(span, self)
 
-          # Publish :trace_finished event
-          events.trace_finished.publish(self) if finished?
-        rescue StandardError => e
-          logger.debug { "Error finishing span on trace: #{e} Backtrace: #{e.backtrace.first(3)}" }
-        end
+        # Publish :trace_finished event
+        events.trace_finished.publish(self) if finished?
+      rescue => e
+        logger.debug { "Error finishing span on trace: #{e.class}: #{e.message} Backtrace: #{e.backtrace.first(3)}" }
       end
 
-      # Track the root span
-      def set_root_span!(span)
+      # Track the root {SpanOperation} object from the current execution context.
+      def set_local_root_span!(span)
         return if span.nil? || root_span
 
         @root_span = span
@@ -527,7 +626,7 @@ module Datadog
           service: service,
           tags: meta,
           metrics: metrics,
-          root_span_id: !partial ? root_span && root_span.id : nil,
+          root_span_id: (!partial) ? root_span&.id : nil,
           profiling_enabled: @profiling_enabled,
           apm_tracing_enabled: @apm_tracing_enabled
         )
@@ -540,12 +639,24 @@ module Datadog
         meta.select { |name, _| name.start_with?(Metadata::Ext::Distributed::TAGS_PREFIX) }
       end
 
+      def reconsider_rule_sample?
+        decision = get_tag(Metadata::Ext::Distributed::TAG_DECISION_MAKER)
+
+        return false if remote_parent || @propagated || @flushed
+        return false unless AUTO_SAMPLING_PRIORITIES.include?(@sampling_priority)
+        return false if decision && !RECONSIDERABLE_DECISIONS.include?(decision)
+
+        true
+      end
+
       def reset
         @root_span = nil
         @active_span = nil
         @active_span_count = 0
         @finished = false
         @spans = []
+        @flushed = false
+        @propagated = false
       end
     end
   end

data/lib/datadog/tracing/trace_segment.rb

@@ -68,8 +68,8 @@ module Datadog
 
         # Does not make an effort to move metrics out of tags
         # The caller is expected to have done that
-        @meta = (tags && tags.dup) || {}
-        @metrics = (metrics && metrics.dup) || {}
+        @meta = tags&.dup || {}
+        @metrics = metrics&.dup || {}
 
         # Set well-known tags, defaulting to getting the values from tags
         @agent_sample_rate = agent_sample_rate || agent_sample_rate_tag