datadog 2.38.0 → 2.47.0

data/lib/datadog/tracing/transport/native.rb

@@ -0,0 +1,395 @@
+# frozen_string_literal: true
+
+require "json"
+require_relative "trace_formatter"
+require_relative "statistics"
+require_relative "span_events_negotiation"
+require_relative "native/telemetry"
+
+module Datadog
+  module Tracing
+    module Transport
+      # Transport backed by the native trace exporter (Rust via C FFI).
+      #
+      # Converts Ruby Span objects directly to Rust spans and delegates
+      # serialization and sending to the Rust data pipeline, which handles
+      # stats computation, msgpack encoding, and HTTP transport with retry
+      # logic.
+      #
+      # Implements the same +send_traces+ / +stats+ interface as
+      # {Datadog::Tracing::Transport::Traces::Transport} so it can be used
+      # as a drop-in replacement via the +Writer+'s +:transport+ option.
+      module Native
+        # Returns +nil+ when the native extension is available, or a +String+
+        # describing why it is not.
+        UNSUPPORTED_REASON = begin
+          require "datadog/core"
+          Datadog::Core::LIBDATADOG_API_FAILURE
+        rescue => e
+          e.message
+        end
+
+        def self.supported?
+          UNSUPPORTED_REASON.nil?
+        end
+
+        # +TraceExporter+ and +TracerSpan+ are defined by the C extension in
+        # +ext/libdatadog_api/trace_exporter.c+ and become available after the
+        # native extension is loaded. +Response+ is a plain Ruby class (see
+        # +native/response.rb+); the C side only resolves and instantiates it.
+        #
+        # The hierarchy is:
+        #   Datadog::Tracing::Transport::Native::TraceExporter (C)
+        #   Datadog::Tracing::Transport::Native::TracerSpan (C)
+        #   Datadog::Tracing::Transport::Native::Response (Ruby)
+
+        # Drop-in transport that delegates to the native trace exporter.
+        class Transport
+          include Statistics
+          include SpanEventsNegotiation
+
+          attr_reader :logger, :url
+
+          def telemetry=(client)
+            @send_mutex.synchronize do
+              if (reporter = @telemetry)
+                reporter.client = client
+              elsif @exporter
+                @telemetry = Telemetry.new(client)
+              end
+            end
+          end
+
+          # @param agent_settings [Datadog::Core::Configuration::AgentSettingsResolver::AgentSettings]
+          #   Agent connection settings, including the request timeout.
+          # @param logger [Logger]
+          def initialize(agent_settings:, logger:)
+            unless Native.supported?
+              raise "Native transport is not supported: #{UNSUPPORTED_REASON}"
+            end
+
+            @logger = logger
+            @telemetry = nil
+
+            # Serializes native sends and is held across a fork. See the
+            # fork-safety note below.
+            @send_mutex = Mutex.new
+
+            # Serializes concurrent fork-hook lifecycles. Unlike @send_mutex,
+            # this is held across the fork through the matching completion hook.
+            @fork_mutex = Mutex.new
+
+            @url = agent_settings.url
+            tracer_version = tracer_version_string
+            language = Core::Environment::Ext::LANG
+            language_version = Core::Environment::Ext::LANG_VERSION
+            language_interpreter = Core::Environment::Ext::LANG_INTERPRETER
+            hostname = begin
+              Core::Environment::Socket.hostname
+            rescue
+              nil
+            end
+            env = Datadog.configuration.env
+            service = Datadog.configuration.service
+            version = Datadog.configuration.version
+
+            exporter = Native::TraceExporter._native_new(
+              url: url,
+              timeout_milliseconds: agent_settings.timeout_seconds * 1000,
+              tracer_version: tracer_version,
+              language: language,
+              language_version: language_version,
+              language_interpreter: language_interpreter,
+              hostname: hostname,
+              env: env,
+              service: service,
+              version: version,
+              client_computed_stats: !Datadog.configuration.apm.tracing.enabled
+            )
+            @exporter = exporter
+
+            # Fork safety: the native exporter owns a long-lived tokio runtime
+            # with background worker threads. Around a fork we must quiesce the
+            # runtime before forking and restore it afterwards in both the parent
+            # and the child (where the inherited runtime is dead).
+            #
+            # A libdatadog Rust call must NOT be interrupted by `fork()`. A
+            # native send releases the GVL during the Rust
+            # `ddog_trace_exporter_send_trace_chunks` call, so if another thread
+            # forks while a (typically the writer/`AsyncTransport`) thread is
+            # mid-send, the child would inherit a half-completed send and
+            # Rust-internal locks, deadlocking or crashing. `_native_before_fork`
+            # also tears down and replaces the runtime, so it must not run while
+            # a send is still using that runtime.
+            #
+            # `@fork_mutex` serializes concurrent forks for this exporter. The
+            # `:before` hook locks it first, then calls `_native_before_fork`,
+            # and finally locks `@send_mutex`, which serializes sends and is
+            # held across the fork. Sends and #close only acquire @send_mutex,
+            # so there is no lock-order inversion.
+            #
+            # `_native_before_fork` must run before locking `@send_mutex` to
+            # pause the runtime before waiting for an in-flight send to drain.
+            # The hook first calls
+            # `_native_before_fork` to pause the runtime, THEN locks the mutex.
+            # This ordering minimizes the serialized section and is safe to run
+            # concurrently with an in-flight send: `block_on` (used by
+            # `send_trace_chunks`) only briefly locks the runtime mutex to clone
+            # the `Arc<Runtime>` (or build a throwaway `current_thread` runtime
+            # if it has been taken) and then drives the send on its own clone
+            # holding no mutex, while `_native_before_fork` just `take()`s the
+            # shared `Arc` and pauses workers -- so neither deadlocks the other.
+            # Locking the mutex AFTER `_native_before_fork` still guarantees the
+            # drain: it BLOCKS until any in-flight send finishes (bounded by the
+            # exporter's request timeout), which drops the send's last `Arc` and
+            # fully shuts the runtime down, and prevents new sends from starting.
+            # The lock is released in the `:parent` and `:child` hooks. In
+            # practice only one writer thread sends, so serializing adds no real
+            # contention.
+            #
+            # Note: the `:before` block runs before genuine forks (web-server
+            # workers, `Process.daemon`) that go through `Process._fork`. It
+            # does NOT run for `system`/backtick/`IO.popen`/`Process.spawn`,
+            # which spawn via `posix_spawn`/`vfork`+`exec` and replace the
+            # process image rather than carrying over the parent's work. Keeping
+            # it cheap is still worthwhile since real forks can be frequent.
+            #
+            # The hooks below are process-global and are never auto-removed by
+            # the fork machinery, so each transport must deregister its own
+            # hooks when it goes away (see #close and the finalizer): otherwise
+            # the closures keep the exporter -- and its runtime threads -- alive
+            # forever, and every later fork runs them against every
+            # historically-created exporter. The closures intentionally close
+            # over only locals (NOT `self`), so the Transport stays GC-eligible and
+            # its finalizer can fire even when #close was not called explicitly.
+            send_mutex = @send_mutex
+            fork_mutex = @fork_mutex
+            before_hook = proc do
+              fork_mutex.lock
+
+              begin
+                # Pause the runtime first (safe to run concurrently with an
+                # in-flight send), then wait for it to drain by acquiring the
+                # mutex. The lock must happen even if `_native_before_fork` raises, so the
+                # in-flight send has returned before the fork. Held across the
+                # fork; released in :parent/:child.
+                exporter._native_before_fork
+              rescue => e
+                Datadog.logger.warn { "Native transport before-fork preparation failed; traces may not be sent to Datadog: #{e.class}: #{e.message}" }
+              ensure
+                send_mutex.lock
+              end
+            end
+            parent_hook = proc do
+              exporter._native_after_fork_in_parent if fork_mutex.owned?
+            rescue => e
+              Datadog.logger.warn { "Native transport after-fork reset failed; traces may not be sent to Datadog: #{e.class}: #{e.message}" }
+            ensure
+              # `:before`'s ensure always locks the mutex on the forking thread
+              # before any fork outcome, so it is normally owned here; the guard
+              # only covers `:before` being interrupted mid-lock (e.g. a kill
+              # between `_native_before_fork` and the lock). Released even if the
+              # native call raised, so a failure can't leave the mutex locked.
+              send_mutex.unlock if send_mutex.owned?
+              fork_mutex.unlock if fork_mutex.owned?
+            end
+            child_hook = proc do
+              exporter._native_after_fork_in_child if fork_mutex.owned?
+            rescue => e
+              Datadog.logger.warn { "Native transport after-fork reset failed; traces may not be sent to Datadog: #{e.class}: #{e.message}" }
+            ensure
+              # See the :parent hook: normally owned (the forking thread is the
+              # lone survivor in the child); the guard only covers `:before`
+              # being interrupted mid-lock.
+              send_mutex.unlock if send_mutex.owned?
+              fork_mutex.unlock if fork_mutex.owned?
+            end
+
+            @fork_hooks = Core::Utils::AtForkMonkeyPatch.at_fork_blocks(
+              before: before_hook,
+              parent: parent_hook,
+              child: child_hook
+            )
+            remove_fork_hooks = self.class.send(:fork_hooks_remover, @fork_hooks)
+
+            # Fallback so a dropped (un-#close'd) transport doesn't leak its
+            # global fork hooks (and, through them, the exporter/runtime). The
+            # removal proc is built by a class method capturing ONLY the
+            # stage->block handles, never `self` -- a finalizer that closed over
+            # the object it is attached to would keep that object reachable and
+            # never run. A fork that already copied these callbacks still owns
+            # them through its matching completion stage.
+            ObjectSpace.define_finalizer(
+              self,
+              remove_fork_hooks
+            )
+          end
+
+          # Deregister this transport's process-global fork hooks and release the
+          # native exporter so its runtime can shut down. Idempotent: safe to
+          # call multiple times and safe to call after the finalizer has run.
+          def close
+            @fork_mutex.synchronize do
+              @send_mutex.synchronize do
+                hooks = @fork_hooks
+                return if hooks.nil?
+
+                begin
+                  @exporter&._native_close
+                ensure
+                  @fork_hooks = nil
+                  @telemetry = nil
+                  @exporter = nil
+                  hooks.each do |stage, block|
+                    Core::Utils::AtForkMonkeyPatch.remove_at_fork(stage, block)
+                  end
+                  ObjectSpace.undefine_finalizer(self)
+                end
+              end
+            end
+
+            nil
+          end
+
+          # Builds the proc that deregisters a transport's fork hooks.
+          #
+          # Defined as a class method so the proc closes over ONLY the
+          # stage->block handles passed in and never over a Transport instance
+          # (which would keep the instance reachable and prevent the finalizer
+          # from ever running). Removing the hooks unpins the exporter captured
+          # by those closures, allowing it (and its runtime) to be freed.
+          def self.fork_hooks_remover(fork_hooks)
+            proc do
+              fork_hooks.each do |stage, block|
+                Core::Utils::AtForkMonkeyPatch.remove_at_fork(stage, block)
+              end
+            end
+          end
+          private_class_method :fork_hooks_remover
+
+          # Send a list of traces to the agent.
+          #
+          # Each trace is a {Datadog::Tracing::TraceSegment} whose +#spans+
+          # returns an +Array+ of {Datadog::Tracing::Span}.
+          #
+          # @param traces [Array<Datadog::Tracing::TraceSegment>]
+          # @return [Array<Response>] one response per batch sent
+          def send_traces(traces)
+            return [] if traces.empty?
+
+            # Apply trace-level tags to root spans (same as the HTTP transport)
+            traces.each { |trace| TraceFormatter.format!(trace, transport: "native") }
+
+            # Build the Array<Array<Span>> structure expected by the C extension.
+            # Each trace segment becomes one inner array (one trace chunk).
+            chunks = traces.map(&:spans)
+
+            native_events_supported = native_events_supported?
+            apply_legacy_span_events!(chunks, native_events_supported)
+
+            # Serialize the native send and hold the mutex across it so a
+            # concurrent fork's :before hook blocks until this send drains
+            # (and `_native_before_fork` cannot tear down the runtime mid-send).
+            # `Mutex#synchronize` releases on exception / `rb_jump_tag` via its
+            # ensure, so interrupt propagation stays correct.
+            responses = @send_mutex.synchronize do
+              # Resolve the exporter under the same lock used by #close, so
+              # close either drains this send or prevents it from starting.
+              exporter = @exporter
+              raise "Native transport has been closed" if exporter.nil?
+
+              exporter._native_send_traces(chunks, native_events_supported) do |observations|
+                @telemetry&.record(observations)
+              end
+            end
+
+            # Update statistics from the response
+            responses.each { |response| update_stats_from_response!(response) }
+
+            responses
+          rescue => e
+            logger.debug { "Native transport error: #{e.class} #{e.message}" }
+            update_stats_from_exception!(e)
+            [InternalErrorResponse.new(e)]
+          end
+
+          private
+
+          # Writes each span's events into the legacy JSON +events+ meta tag
+          # when the agent lacks typed-event support, mutating spans in place.
+          #
+          # @param chunks [Array<Array<Datadog::Tracing::Span>>] trace chunks to serialize
+          # @param native_events_supported [Boolean] whether the agent accepts typed events
+          # @return [void]
+          def apply_legacy_span_events!(chunks, native_events_supported)
+            return if native_events_supported
+
+            chunks.each do |spans|
+              spans.each do |span|
+                next if span.events.empty?
+
+                span.set_tag("events", span.events.map(&:to_hash).to_json)
+              end
+            end
+
+            nil
+          end
+
+          def tracer_version_string
+            defined?(Datadog::VERSION::STRING) ? Datadog::VERSION::STRING : "unknown"
+          end
+        end
+
+        # Response for internal errors (exceptions raised before reaching
+        # the native transport).
+        class InternalErrorResponse
+          attr_reader :error
+
+          def initialize(error)
+            @error = error
+          end
+
+          def ok?
+            false
+          end
+
+          def internal_error?
+            true
+          end
+
+          def server_error?
+            false
+          end
+
+          def client_error?
+            false
+          end
+
+          def not_found?
+            false
+          end
+
+          def unsupported?
+            false
+          end
+
+          def payload
+            nil
+          end
+
+          def trace_count
+            0
+          end
+
+          def service_rates
+            nil
+          end
+
+          def inspect
+            "#<#{self.class} error=#{error.inspect}>"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/transport/serializable_trace.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-require 'json'
-require 'msgpack'
-require 'datadog/tracing/utils'
+require "json"
+require "msgpack"
+require "datadog/tracing/utils"
 
 module Datadog
   module Tracing
@@ -76,10 +76,10 @@ module Datadog
           if span.stopped?
             packer.write_map_header(number_of_elements_to_write + 2) # Set header with how many elements in the map
 
-            packer.write('start')
+            packer.write("start")
             packer.write(time_nano(span.start_time))
 
-            packer.write('duration')
+            packer.write("duration")
             packer.write(duration_nano(span.duration))
           else
             packer.write_map_header(number_of_elements_to_write) # Set header with how many elements in the map
@@ -88,40 +88,40 @@ module Datadog
           if span.events.any?
             if @native_events_supported
               # Use top-level field for native events
-              packer.write('span_events')
+              packer.write("span_events")
               packer.write(span.events.map(&:to_native_format))
             else
               # Serialize span events as meta tags
-              span.set_tag('events', span.events.map(&:to_hash).to_json)
+              span.set_tag("events", span.events.map(&:to_hash).to_json)
             end
           end
 
           # DEV: We use strings as keys here, instead of symbols, as
           # DEV: MessagePack will ultimately convert them to strings.
           # DEV: By providing strings directly, we skip this indirection operation.
-          packer.write('span_id')
+          packer.write("span_id")
           packer.write(span.id)
-          packer.write('parent_id')
+          packer.write("parent_id")
           packer.write(span.parent_id)
-          packer.write('trace_id')
+          packer.write("trace_id")
           packer.write(@trace_id)
-          packer.write('name')
+          packer.write("name")
           packer.write(span.name)
-          packer.write('service')
+          packer.write("service")
           packer.write(span.service)
-          packer.write('resource')
+          packer.write("resource")
           packer.write(span.resource)
-          packer.write('type')
+          packer.write("type")
           packer.write(span.type)
-          packer.write('meta')
+          packer.write("meta")
           packer.write(span.meta)
-          packer.write('metrics')
+          packer.write("metrics")
           packer.write(span.metrics)
-          packer.write('meta_struct')
+          packer.write("meta_struct")
           packer.write(span.metastruct)
-          packer.write('span_links')
+          packer.write("span_links")
           packer.write(span.links.map(&:to_hash))
-          packer.write('error')
+          packer.write("error")
           packer.write(span.status)
           packer
         end

data/lib/datadog/tracing/transport/span_events_negotiation.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Tracing
+    module Transport
+      # Shared agent capability negotiation for span event wire formats.
+      module SpanEventsNegotiation
+        private
+
+        # Queries whether the agent accepts typed span events, which selects
+        # between the typed field and legacy JSON metadata. Only successful
+        # capability responses are cached so a later flush can recover.
+        #
+        # The memo is read and written without synchronization, independent of
+        # any lock the including transport holds for its own sends (the native
+        # transport's +@send_mutex+, for example). Two concurrent sends can both
+        # observe it unset and each issue an +agent_info.fetch+; the duplicate
+        # fetch is self-correcting because the last writer wins.
+        #
+        # @return [Boolean] true if typed span events are supported
+        def native_events_supported?
+          return @native_events_supported if defined?(@native_events_supported)
+
+          option = Datadog.configuration.tracing.native_span_events
+          unless option.nil?
+            @native_events_supported = option
+            return option
+          end
+
+          components = Datadog.send(:components, allow_initialization: false)
+          return false unless components
+
+          response = components.agent_info.fetch
+          return false unless response
+
+          @native_events_supported = response.span_events == true
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/transport/statistics.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require_relative '../../core/metrics/metric'
-require_relative '../../core/diagnostics/health'
+require_relative "../../core/metrics/metric"
+require_relative "../../core/diagnostics/health"
 
 module Datadog
   module Tracing

data/lib/datadog/tracing/transport/telemetry.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Tracing
+    module Transport
+      class Telemetry
+        # Observation schema: Symbol keys with non-negative Integer values.
+        # COUNTS maps per-send count deltas to [metric name, additional tags].
+        # :bytes_sent is one successful payload's byte-size distribution sample;
+        # record each payload separately rather than summing its samples.
+        # :responses_count requires :status_code; omit both when the code is unknown.
+        # Missing and zero measurements emit nothing. All metrics use namespace
+        # "tracers" and common: true; trace_* metrics gain src_library:<source>, while
+        # span metrics keep their source-free tags.
+        COUNTS = {
+          requests_count: ["trace_api.requests", []],
+          errors_network: ["trace_api.errors", ["type:network"]],
+          errors_timeout: ["trace_api.errors", ["type:timeout"]],
+          errors_status_code: ["trace_api.errors", ["type:status_code"]],
+          chunks_sent: ["trace_chunks_sent", []],
+          chunks_dropped_serialization_error: ["trace_chunks_dropped", ["reason:serialization_error"]],
+          chunks_dropped_send_failure: ["trace_chunks_dropped", ["reason:send_failure"]],
+          chunks_dropped_payload_too_large: ["trace_chunks_dropped", ["reason:payload_too_large"]],
+          spans_enqueued_for_serialization: ["spans_enqueued_for_serialization", []],
+          spans_dropped_serialization_error: ["spans_dropped", ["reason:serialization_error"]],
+          spans_dropped_api_error: ["spans_dropped", ["reason:api_error"]],
+          spans_dropped_payload_too_large: ["spans_dropped", ["reason:payload_too_large"]],
+        }.freeze #: Hash[Symbol, [String, Array[String]]]
+
+        attr_writer :client
+
+        def initialize(client, source:)
+          @client = client
+          @source_tags = ["src_library:#{source}"]
+          @counts = COUNTS.map do |key, (name, tags)|
+            [key, name, name.start_with?("trace_") ? @source_tags + tags : tags]
+          end
+        end
+
+        def record(observations)
+          client = @client
+          return unless client.enabled?
+
+          @counts.each do |key, name, tags|
+            value = observations.fetch(key, 0)
+            client.inc("tracers", name, value, tags: tags) if value > 0
+          end
+          bytes = observations.fetch(:bytes_sent, 0)
+          client.distribution("tracers", "trace_api.bytes", bytes, tags: @source_tags) if bytes > 0
+          responses = observations.fetch(:responses_count, 0)
+          if responses > 0
+            client.inc("tracers", "trace_api.responses", responses,
+              tags: @source_tags + ["status_code:#{observations.fetch(:status_code)}"],)
+          end
+          nil
+        rescue
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/transport/trace_formatter.rb

@@ -1,13 +1,13 @@
 # frozen_string_literal: true
 
-require_relative '../../core/environment/identity'
-require_relative '../../core/environment/process'
-require_relative '../../core/environment/socket'
-require_relative '../../core/environment/git'
-require_relative '../../core/git/ext'
-require_relative '../../core/runtime/ext'
-require_relative '../metadata/ext'
-require_relative '../trace_segment'
+require_relative "../../core/environment/identity"
+require_relative "../../core/environment/process"
+require_relative "../../core/environment/socket"
+require_relative "../../core/environment/git"
+require_relative "../../core/git/ext"
+require_relative "../../core/runtime/ext"
+require_relative "../metadata/ext"
+require_relative "../trace_segment"
 
 module Datadog
   module Tracing
@@ -19,8 +19,8 @@ module Datadog
           :first_span,
           :trace
 
-        def self.format!(trace)
-          new(trace).format!
+        def self.format!(trace, transport: nil)
+          new(trace).format!(transport: transport)
         end
 
         def initialize(trace)
@@ -32,7 +32,7 @@ module Datadog
         end
 
         # Modifies a trace so suitable for transport
-        def format!
+        def format!(transport: nil)
           return unless trace
           return trace unless root_span
 
@@ -65,10 +65,13 @@ module Datadog
 
           if first_span
             tag_process_tags!
+            tag_sdk_otlp_export!
             tag_git_repository_url!
             tag_git_commit_sha!
           end
 
+          root_span.set_tag("_dd.tracing.transport", transport) if transport && !partial?
+
           trace
         end
 
@@ -117,7 +120,7 @@ module Datadog
 
           root_span.set_tag(
             Tracing::Metadata::Ext::Distributed::TAG_KNUTH_SAMPLING_RATE,
-            format('%.6f', (rate * 1e6).round / 1e6).sub(/\.?0+\z/, '')
+            format("%.6f", (rate * 1e6).round / 1e6).sub(/\.?0+\z/, "")
           )
         end
 
@@ -213,7 +216,7 @@ module Datadog
         def tag_apm_tracing_disabled!
           return if trace.apm_tracing_enabled
 
-          root_span.set_tag(Tracing::Metadata::Ext::TAG_APM_ENABLED, 0)
+          trace.spans.each { |span| span.set_metric(Tracing::Metadata::Ext::TAG_APM_ENABLED, 0) }
         end
 
         def tag_git_repository_url!
@@ -237,6 +240,13 @@ module Datadog
           )
         end
 
+        # TODO: Revisit when OTLP trace export support lands. The OTLP path sets
+        # `_dd.sdk.otlp_export: "true"` on the OTLP resource, so this tag must not
+        # also be set to "false" on the first span of a chunk exported over OTLP.
+        def tag_sdk_otlp_export!
+          first_span.set_tag(Tracing::Metadata::Ext::TAG_SDK_OTLP_EXPORT, "false")
+        end
+
         private
 
         def partial?