datadog 2.17.0 → 2.18.0

data/lib/datadog/di/serializer.rb

@@ -82,7 +82,11 @@ module Datadog
       # between positional and keyword arguments. We convert positional
       # arguments to keyword arguments ("arg1", "arg2", ...) and ensure
       # the positional arguments are listed first.
-      def serialize_args(args, kwargs,
+      #
+      # Instance variables are technically a hash just like kwargs,
+      # we take them as a separate parameter to avoid a hash merge
+      # in upstream code.
+      def serialize_args(args, kwargs, instance_vars,
         depth: settings.dynamic_instrumentation.max_capture_depth,
         attribute_count: settings.dynamic_instrumentation.max_capture_attribute_count)
         counter = 0
@@ -91,7 +95,7 @@ module Datadog
           # Conversion to symbol is needed here to put args ahead of
           # kwargs when they are merged below.
           c[:"arg#{counter}"] = value
-        end.update(kwargs)
+        end.update(kwargs).update(instance_vars)
         serialize_vars(combined, depth: depth, attribute_count: attribute_count)
       end
 

data/lib/datadog/di/transport/http/input.rb

@@ -53,6 +53,16 @@ module Datadog
                 # Encode body & type
                 env.headers[HEADER_CONTENT_TYPE] = encoder.content_type
                 env.body = env.request.parcel.data
+                env.query = {
+                  # DEV: In theory we could serialize the tags here
+                  # rather than requiring them to be pre-serialized.
+                  # In practice the tags should be relatively static
+                  # (they would change when process forks, and hostname
+                  # could change at any time but probably we should ignore
+                  # those changes), therefore serializing the tags
+                  # every time would be wasteful.
+                  ddtags: env.request.serialized_tags,
+                }
 
                 super
               end

data/lib/datadog/di/transport/input.rb

@@ -12,6 +12,13 @@ module Datadog
         end
 
         class Request < Datadog::Core::Transport::Request
+          attr_reader :serialized_tags
+
+          def initialize(parcel, serialized_tags)
+            super(parcel)
+
+            @serialized_tags = serialized_tags
+          end
         end
 
         class Transport
@@ -28,10 +35,11 @@ module Datadog
             @apis[HTTP::API::INPUT]
           end
 
-          def send_input(payload)
+          def send_input(payload, tags)
             json = JSON.dump(payload)
             parcel = EncodedParcel.new(json)
-            request = Request.new(parcel)
+            serialized_tags = Core::TagBuilder.serialize_tags(tags)
+            request = Request.new(parcel, serialized_tags)
 
             response = @client.send_input_payload(request)
             unless response.ok?

data/lib/datadog/profiling/collectors/code_provenance.rb

@@ -14,7 +14,10 @@ module Datadog
       #
       # This class acts both as a collector (collecting data) as well as a recorder (records/serializes it)
       class CodeProvenance
-        def initialize(standard_library_path: RbConfig::CONFIG.fetch("rubylibdir"))
+        def initialize(
+          standard_library_path: RbConfig::CONFIG.fetch("rubylibdir"),
+          ruby_native_filename: Datadog::Profiling::Collectors::Stack._native_ruby_native_filename
+        )
           @libraries_by_name = {}
           @libraries_by_path = {}
           @seen_files = Set.new
@@ -26,6 +29,7 @@ module Datadog
               name: "stdlib",
               version: RUBY_VERSION,
               path: standard_library_path,
+              extra_path: ruby_native_filename,
             )
           )
         end
@@ -37,10 +41,6 @@ module Datadog
           self
         end
 
-        def generate
-          seen_libraries
-        end
-
         def generate_json
           JSON.generate(v1: seen_libraries.to_a)
         end
@@ -79,7 +79,15 @@ module Datadog
           loaded_specs.each do |spec|
             next if libraries_by_name.key?(spec.name)
 
-            record_library(Library.new(kind: "library", name: spec.name, version: spec.version, path: spec.gem_dir))
+            record_library(
+              Library.new(
+                kind: "library",
+                name: spec.name,
+                version: spec.version,
+                path: spec.gem_dir,
+                extra_path: (spec.extension_dir if spec.extensions.any?),
+              )
+            )
             recorded_library = true
           end
 
@@ -110,11 +118,12 @@ module Datadog
         class Library
           attr_reader :kind, :name, :version
 
-          def initialize(kind:, name:, version:, path:)
+          def initialize(kind:, name:, version:, path:, extra_path: nil)
+            extra_path = nil if extra_path&.empty?
             @kind = kind.freeze
             @name = name.dup.freeze
             @version = version.to_s.dup.freeze
-            @paths = [path.dup.freeze].freeze
+            @paths = [path.dup.freeze, extra_path.dup.freeze].compact.freeze
             freeze
           end
 

data/lib/datadog/profiling/collectors/cpu_and_wall_time_worker.rb

@@ -33,6 +33,9 @@ module Datadog
             Datadog.logger.warn(
               "Profiling dynamic sampling rate disabled. This should only be used for testing, and will increase overhead!"
             )
+            Datadog::Core::Telemetry::Logger.error(
+              "Profiling dynamic sampling rate disabled. This should only be used for testing, and will increase overhead!"
+            )
           end
 
           self.class._native_initialize(
@@ -77,6 +80,7 @@ module Datadog
                 "Cause: #{e.class.name} #{e.message} Location: #{Array(e.backtrace).first}"
               )
               on_failure_proc&.call
+              Datadog::Core::Telemetry::Logger.report(e, description: "CpuAndWallTimeWorker thread error", pii_safe: true)
             end
             @worker_thread.name = self.class.name # Repeated from above to make sure thread gets named asap
             @worker_thread.thread_variable_set(:fork_safe, true)

data/lib/datadog/profiling/collectors/idle_sampling_helper.rb

@@ -41,6 +41,7 @@ module Datadog
                 "IdleSamplingHelper thread error. " \
                 "Cause: #{e.class.name} #{e.message} Location: #{Array(e.backtrace).first}"
               )
+              Datadog::Core::Telemetry::Logger.report(e, description: "IdleSamplingHelper thread error", pii_safe: true)
             end
             @worker_thread.name = self.class.name # Repeated from above to make sure thread gets named asap
             @worker_thread.thread_variable_set(:fork_safe, true)

data/lib/datadog/profiling/collectors/thread_context.rb

@@ -21,7 +21,8 @@ module Datadog
           endpoint_collection_enabled:,
           timeline_enabled:,
           waiting_for_gvl_threshold_ns:,
-          otel_context_enabled:
+          otel_context_enabled:,
+          native_filenames_enabled:
         )
           tracer_context_key = safely_extract_context_key_from(tracer)
           self.class._native_initialize(
@@ -33,6 +34,7 @@ module Datadog
             timeline_enabled: timeline_enabled,
             waiting_for_gvl_threshold_ns: waiting_for_gvl_threshold_ns,
             otel_context_enabled: otel_context_enabled,
+            native_filenames_enabled: validate_native_filenames(native_filenames_enabled),
           )
         end
 
@@ -44,6 +46,7 @@ module Datadog
           timeline_enabled: false,
           waiting_for_gvl_threshold_ns: 10_000_000,
           otel_context_enabled: false,
+          native_filenames_enabled: true,
           **options
         )
           new(
@@ -54,6 +57,7 @@ module Datadog
             timeline_enabled: timeline_enabled,
             waiting_for_gvl_threshold_ns: waiting_for_gvl_threshold_ns,
             otel_context_enabled: otel_context_enabled,
+            native_filenames_enabled: native_filenames_enabled,
             **options,
           )
         end
@@ -81,6 +85,17 @@ module Datadog
           context = provider.instance_variable_get(:@context)
           context&.instance_variable_get(:@key)
         end
+
+        def validate_native_filenames(native_filenames_enabled)
+          if native_filenames_enabled && !Datadog::Profiling::Collectors::Stack._native_filenames_available?
+            Datadog.logger.debug(
+              "Native filenames are enabled, but the required dladdr API was not available. Disabling native filenames."
+            )
+            false
+          else
+            native_filenames_enabled
+          end
+        end
       end
     end
   end

data/lib/datadog/profiling/component.rb

@@ -96,6 +96,7 @@ module Datadog
           timeline_enabled: timeline_enabled,
           waiting_for_gvl_threshold_ns: settings.profiling.advanced.waiting_for_gvl_threshold_ns,
           otel_context_enabled: settings.profiling.advanced.preview_otel_context_enabled,
+          native_filenames_enabled: settings.profiling.advanced.native_filenames_enabled,
         )
       end
 
@@ -221,13 +222,14 @@ module Datadog
         end
 
         unless allocation_profiling_enabled
-          raise ArgumentError, "Heap profiling requires allocation profiling to be enabled"
+          logger.warn(
+            "Heap profiling was requested but allocation profiling is not enabled. " \
+            "Heap profiling has been disabled."
+          )
+          return false
         end
 
-        logger.warn(
-          "Enabled experimental heap profiling: heap_sample_rate=#{heap_sample_rate}. This is experimental, not " \
-          "recommended, and will increase overhead!"
-        )
+        logger.debug("Enabled heap profiling: heap_sample_rate=#{heap_sample_rate}")
 
         true
       end
@@ -237,10 +239,6 @@ module Datadog
 
         return false unless heap_profiling_enabled && heap_size_profiling_enabled
 
-        logger.warn(
-          "Enabled experimental heap size profiling. This is experimental, not recommended, and will increase overhead!"
-        )
-
         true
       end
 

data/lib/datadog/profiling/ext.rb

@@ -11,19 +11,7 @@ module Datadog
 
       module Transport
         module HTTP
-          FORM_FIELD_TAG_ENV = "env"
-          FORM_FIELD_TAG_HOST = "host"
-          FORM_FIELD_TAG_LANGUAGE = "language"
-          FORM_FIELD_TAG_PID = "process_id"
           FORM_FIELD_TAG_PROFILER_VERSION = "profiler_version"
-          FORM_FIELD_TAG_RUNTIME = "runtime"
-          FORM_FIELD_TAG_RUNTIME_ENGINE = "runtime_engine"
-          FORM_FIELD_TAG_RUNTIME_ID = "runtime-id"
-          FORM_FIELD_TAG_RUNTIME_VERSION = "runtime_version"
-          FORM_FIELD_TAG_SERVICE = "service"
-          FORM_FIELD_TAG_VERSION = "version"
-          TAG_GIT_REPOSITORY_URL = "git.repository_url"
-          TAG_GIT_COMMIT_SHA = "git.commit.sha"
 
           CODE_PROVENANCE_FILENAME = "code-provenance.json"
         end

data/lib/datadog/profiling/http_transport.rb

@@ -37,7 +37,7 @@ module Datadog
             Datadog.logger.debug("Successfully reported profiling data")
             true
           else
-            Datadog.logger.error(
+            Datadog.logger.warn(
               "Failed to report profiling data (#{config_without_api_key}): " \
               "server returned unexpected HTTP #{result} status code"
             )
@@ -47,7 +47,7 @@ module Datadog
             false
           end
         else
-          Datadog.logger.error("Failed to report profiling data (#{config_without_api_key}): #{result}")
+          Datadog.logger.warn("Failed to report profiling data (#{config_without_api_key}): #{result}")
           Datadog::Core::Telemetry::Logger.error("Failed to report profiling data")
           false
         end

data/lib/datadog/profiling/profiler.rb

@@ -50,6 +50,8 @@ module Datadog
           "Detected issue with profiler (#{failed_component} component), stopping profiling. " \
           "See previous log messages for details."
         )
+        Datadog::Core::Telemetry::Logger
+          .error("Detected issue with profiler (#{failed_component} component), stopping profiling")
 
         # We explicitly not stop the crash tracker in this situation, under the assumption that, if a component failed,
         # we're operating in a degraded state and crash tracking may still be helpful.

data/lib/datadog/profiling/scheduler.rb

@@ -68,6 +68,7 @@ module Datadog
           "Cause: #{e.class.name} #{e.message} Location: #{Array(e.backtrace).first}"
         )
         on_failure_proc&.call
+        Datadog::Core::Telemetry::Logger.report(e, description: "Profiling::Scheduler thread error")
         raise
       ensure
         Datadog.logger.debug("#flush was interrupted or failed before it could complete") if interrupted
@@ -133,7 +134,7 @@ module Datadog
         begin
           transport.export(flush)
         rescue => e
-          Datadog.logger.error(
+          Datadog.logger.warn(
             "Unable to report profile. Cause: #{e.class.name} #{e.message} Location: #{Array(e.backtrace).first}"
           )
           Datadog::Core::Telemetry::Logger.report(e, description: "Unable to report profile")

data/lib/datadog/profiling/stack_recorder.rb

@@ -23,7 +23,7 @@ module Datadog
         # This isn't something we expect to happen normally, but because it would break the assumptions of the
         # C-level mutexes (that there is a single serializer thread), we add it here as an extra safeguard against it
         # accidentally happening.
-        @no_concurrent_synchronize_mutex = Mutex.new
+        @no_concurrent_serialize_mutex = Mutex.new
 
         self.class._native_initialize(
           self_instance: self,
@@ -60,7 +60,7 @@ module Datadog
       end
 
       def serialize
-        status, result = @no_concurrent_synchronize_mutex.synchronize { self.class._native_serialize(self) }
+        status, result = @no_concurrent_serialize_mutex.synchronize { self.class._native_serialize(self) }
 
         if status == :ok
           start, finish, encoded_profile, profile_stats = result
@@ -71,15 +71,15 @@ module Datadog
         else
           error_message = result
 
-          Datadog.logger.error("Failed to serialize profiling data: #{error_message}")
-          Datadog::Core::Telemetry::Logger.error("Failed to serialize profiling data")
+          Datadog.logger.warn("Failed to serialize profiling data: #{error_message}")
+          Datadog::Core::Telemetry::Logger.error("Failed to serialize profiling data (#{error_message})")
 
           nil
         end
       end
 
       def serialize!
-        status, result = @no_concurrent_synchronize_mutex.synchronize { self.class._native_serialize(self) }
+        status, result = @no_concurrent_serialize_mutex.synchronize { self.class._native_serialize(self) }
 
         if status == :ok
           _start, _finish, encoded_profile = result

data/lib/datadog/profiling/tag_builder.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
+require_relative "../core/tag_builder"
 require_relative "../core/utils"
-require_relative "../core/environment/git"
 
 module Datadog
   module Profiling
@@ -11,45 +11,13 @@ module Datadog
 
       def self.call(
         settings:,
-        # Unified service tagging
-        env: settings.env,
-        service: settings.service,
-        version: settings.version,
         # Other metadata
-        host: Core::Environment::Socket.hostname,
-        language: Core::Environment::Identity.lang,
-        pid: Process.pid.to_s,
-        profiler_version: Core::Environment::Identity.gem_datadog_version,
-        runtime_engine: Core::Environment::Identity.lang_engine,
-        runtime_id: Core::Environment::Identity.id,
-        runtime_version: Core::Environment::Identity.lang_version,
-        git_repository_url: Core::Environment::Git.git_repository_url,
-        git_commit_sha: Core::Environment::Git.git_commit_sha,
-        # User-provided tags
-        user_tags: settings.tags
+        profiler_version: Core::Environment::Identity.gem_datadog_version
       )
-        tags = {
-          # When changing or adding these, make sure they are kept in sync with
-          # https://docs.google.com/spreadsheets/d/1LOGMf4c4Avbtn36uZ2SWvhIGKRPLM1BoWkUP4JYj7hA/ (Datadog internal link)
-          FORM_FIELD_TAG_HOST => host,
-          FORM_FIELD_TAG_LANGUAGE => language,
-          FORM_FIELD_TAG_PID => pid,
+        hash = Core::TagBuilder.tags(settings).merge(
           FORM_FIELD_TAG_PROFILER_VERSION => profiler_version,
-          FORM_FIELD_TAG_RUNTIME => language, # This is known to be repeated from language, above
-          FORM_FIELD_TAG_RUNTIME_ENGINE => runtime_engine,
-          FORM_FIELD_TAG_RUNTIME_ID => runtime_id,
-          FORM_FIELD_TAG_RUNTIME_VERSION => runtime_version,
-        }
-        tags[FORM_FIELD_TAG_ENV] = env if env
-        tags[FORM_FIELD_TAG_SERVICE] = service if service
-        tags[FORM_FIELD_TAG_VERSION] = version if version
-        tags[TAG_GIT_REPOSITORY_URL] = git_repository_url if git_repository_url
-        tags[TAG_GIT_COMMIT_SHA] = git_commit_sha if git_commit_sha
-
-        # Make sure everything is an utf-8 string, to avoid encoding issues in native code/further downstream
-        user_tags.merge(tags).map do |key, value|
-          [Datadog::Core::Utils.utf8_encode(key), Datadog::Core::Utils.utf8_encode(value)]
-        end.to_h
+        )
+        Core::Utils.encode_tags(hash)
       end
     end
   end

data/lib/datadog/profiling/tasks/setup.rb

@@ -19,6 +19,7 @@ module Datadog
               "Profiler extensions unavailable. Cause: #{e.class.name} #{e.message} " \
               "Location: #{Array(e.backtrace).first}"
             end
+            Datadog::Core::Telemetry::Logger.report(e, description: "Profiler extensions unavailable")
           end
         end
 
@@ -33,6 +34,7 @@ module Datadog
               "Error during post-fork hooks. Cause: #{e.class.name} #{e.message} " \
               "Location: #{Array(e.backtrace).first}"
             end
+            Datadog::Core::Telemetry::Logger.report(e, description: "Error during post-fork hooks")
           end
         end
       end

data/lib/datadog/tracing/contrib/action_pack/action_controller/instrumentation.rb

@@ -68,6 +68,12 @@ module Datadog
 
                 span.set_tag(Ext::TAG_ROUTE_ACTION, payload.fetch(:action))
                 span.set_tag(Ext::TAG_ROUTE_CONTROLLER, payload.fetch(:controller))
+                if (runtime = payload[:view_runtime])
+                  span.set_tag(Ext::TAG_VIEW_RUNTIME, runtime)
+                end
+                if (runtime = payload[:db_runtime])
+                  span.set_tag(Ext::TAG_DB_RUNTIME, runtime)
+                end
 
                 exception = payload[:exception_object]
                 if exception.nil?
@@ -118,6 +124,15 @@ module Datadog
                   payload[:exception] = [e.class.name, e.message]
                   payload[:exception_object] = e
                   raise e
+                ensure
+                  # Database and view runtime are available for controllers
+                  # deriving from ActionController::Base.
+                  # They are not defined on controllers deriving from
+                  # ActionController::Metal, unless
+                  # ActionController::Instrumentation is explicitly included
+                  # into the controller class.
+                  payload[:db_runtime] = db_runtime if respond_to?(:db_runtime)
+                  payload[:view_runtime] = view_runtime if respond_to?(:view_runtime)
                 end
               # rubocop:enable Lint/RescueException
               ensure

data/lib/datadog/tracing/contrib/action_pack/action_dispatch/instrumentation.rb

@@ -9,9 +9,12 @@ module Datadog
         module ActionDispatch
           # Instrumentation for ActionDispatch components
           module Instrumentation
+            SCRIPT_NAME_KEY = 'SCRIPT_NAME'
+            FORMAT_SUFFIX = '(.:format)'
+
             module_function
 
-            def set_http_route_tags(route_spec, script_name)
+            def set_http_route_tags(route_spec, route_path)
               return unless Tracing.enabled?
 
               return unless route_spec
@@ -19,10 +22,10 @@ module Datadog
               request_trace = Tracing.active_trace
               return unless request_trace
 
-              request_trace.set_tag(Tracing::Metadata::Ext::HTTP::TAG_ROUTE, route_spec.to_s.gsub(/\(.:format\)\z/, ''))
+              request_trace.set_tag(Tracing::Metadata::Ext::HTTP::TAG_ROUTE, route_spec)
 
-              if script_name && !script_name.empty?
-                request_trace.set_tag(Tracing::Metadata::Ext::HTTP::TAG_ROUTE_PATH, script_name)
+              if route_path && !route_path.empty?
+                request_trace.set_tag(Tracing::Metadata::Ext::HTTP::TAG_ROUTE_PATH, route_path)
               end
             end
 
@@ -40,16 +43,17 @@ module Datadog
               # Instrumentation for ActionDispatch::Journey::Router for Rails versions older than 7.1
               module Router
                 def find_routes(req)
+                  # result is an array of [match, parameters, route] tuples
                   result = super
+                  result.each do |_, _, route|
+                    next unless Instrumentation.dispatcher_route?(route)
 
-                  # result is an array of [match, parameters, route] tuples
-                  routes = result.map(&:last)
+                    http_route = route.path.spec.to_s
+                    http_route.delete_suffix!(FORMAT_SUFFIX)
 
-                  routes.each do |route|
-                    if Instrumentation.dispatcher_route?(route)
-                      Instrumentation.set_http_route_tags(route.path.spec, req.env['SCRIPT_NAME'])
-                      break
-                    end
+                    Instrumentation.set_http_route_tags(http_route, req.env[SCRIPT_NAME_KEY])
+
+                    break
                   end
 
                   result
@@ -62,7 +66,10 @@ module Datadog
                 def find_routes(req)
                   super do |match, parameters, route|
                     if Instrumentation.dispatcher_route?(route)
-                      Instrumentation.set_http_route_tags(route.path.spec, req.env['SCRIPT_NAME'])
+                      http_route = route.path.spec.to_s
+                      http_route.delete_suffix!(FORMAT_SUFFIX)
+
+                      Instrumentation.set_http_route_tags(http_route, req.env[SCRIPT_NAME_KEY])
                     end
 
                     yield [match, parameters, route]

data/lib/datadog/tracing/contrib/action_pack/ext.rb

@@ -16,6 +16,8 @@ module Datadog
           TAG_OPERATION_CONTROLLER = 'controller'
           TAG_ROUTE_ACTION = 'rails.route.action'
           TAG_ROUTE_CONTROLLER = 'rails.route.controller'
+          TAG_DB_RUNTIME = 'rails.db.runtime'
+          TAG_VIEW_RUNTIME = 'rails.view.runtime'
         end
       end
     end

data/lib/datadog/tracing/contrib/lograge/patcher.rb

@@ -20,10 +20,12 @@ module Datadog
 
           # patch applies our patch
           def patch
+            # First check Lograge logger directly for when keep_original_rails_log option is used
+            used_logger = ::Lograge.logger || ::Lograge::LogSubscribers::ActionController.logger
+
             # ActiveSupport::TaggedLogging is the default Rails logger since Rails 5
             if defined?(::ActiveSupport::TaggedLogging::Formatter) &&
-                ::Lograge::LogSubscribers::ActionController
-                    .logger&.formatter.is_a?(::ActiveSupport::TaggedLogging::Formatter)
+                used_logger&.formatter.is_a?(::ActiveSupport::TaggedLogging::Formatter)
               Datadog.logger.warn(
                 'Lograge and ActiveSupport::TaggedLogging (the default Rails log formatter) are not compatible: ' \
                   'Lograge does not account for Rails log tags, creating polluted logs and breaking log formatting. ' \

data/lib/datadog/tracing/contrib/sidekiq/ext.rb

@@ -13,6 +13,7 @@ module Datadog
           ENV_ANALYTICS_ENABLED = 'DD_TRACE_SIDEKIQ_ANALYTICS_ENABLED'
           ENV_ANALYTICS_SAMPLE_RATE = 'DD_TRACE_SIDEKIQ_ANALYTICS_SAMPLE_RATE'
           SERVICE_NAME = 'sidekiq'
+          SIDEKIQ_8_SECONDS_PER_INTEGER = 0.001 # Sidekiq 8 uses integer epoch milliseconds, rather than epoch floats
           SPAN_PUSH = 'sidekiq.push'
           SPAN_JOB = 'sidekiq.job'
           SPAN_JOB_FETCH = 'sidekiq.job_fetch'

data/lib/datadog/tracing/contrib/sidekiq/server_tracer.rb

@@ -22,7 +22,7 @@ module Datadog
             @quantize = options[:quantize] || configuration[:quantize]
           end
 
-          def call(worker, job, queue)
+          def call(worker, job, queue) # rubocop:disable Metrics/MethodLength
             resource = job_resource(job)
 
             if @distributed_tracing
@@ -61,7 +61,10 @@ module Datadog
               span.set_tag(Ext::TAG_JOB_RETRY_COUNT, job['retry_count'])
               span.set_tag(Ext::TAG_JOB_QUEUE, job['queue'])
               span.set_tag(Ext::TAG_JOB_WRAPPER, job['class']) if job['wrapped']
-              span.set_tag(Ext::TAG_JOB_DELAY, 1000.0 * (Core::Utils::Time.now.utc.to_f - job['enqueued_at'].to_f))
+
+              enqueued_at = job['enqueued_at']
+              enqueued_at *= Ext::SIDEKIQ_8_SECONDS_PER_INTEGER if enqueued_at.is_a?(Integer)
+              span.set_tag(Ext::TAG_JOB_DELAY, 1000.0 * (Core::Utils::Time.now.utc.to_f - enqueued_at.to_f))
 
               args = job['args']
               if args && !args.empty?

data/lib/datadog/tracing/sync_writer.rb

@@ -25,7 +25,7 @@ module Datadog
       # @param [Datadog::Tracing::Transport::Traces::Transport] transport a custom transport instance.
       #   If provided, overrides `transport_options` and `agent_settings`.
       # @param [Hash<Symbol,Object>] transport_options options for the default transport instance.
-      # @param [Datadog::Tracing::Configuration::AgentSettingsResolver::AgentSettings] agent_settings agent options for
+      # @param [Datadog::Tracing::Configuration::AgentSettings] agent_settings agent options for
       #   the default transport instance.
       def initialize(transport: nil, transport_options: {}, agent_settings: nil, logger: Datadog.logger)
         @logger = logger

data/lib/datadog/tracing/trace_operation.rb

@@ -79,7 +79,7 @@ module Datadog
         trace_state: nil,
         trace_state_unknown_fields: nil,
         remote_parent: false,
-        tracer: nil,
+        tracer: nil, # DEV-3.0: deprecated, remove in 3.0
         baggage: nil
       )
         @logger = logger
@@ -106,7 +106,6 @@ module Datadog
         @apm_tracing_enabled = apm_tracing_enabled
         @trace_state = trace_state
         @trace_state_unknown_fields = trace_state_unknown_fields
-        @tracer = tracer
         @baggage = baggage
 
         # Generic tags
@@ -329,7 +328,7 @@ module Datadog
         span_id = @active_span && @active_span.id
         span_id ||= @parent_span_id unless finished?
         # sample the trace_operation with the tracer
-        @tracer&.sample_trace(self) unless sampling_priority
+        events.trace_propagated.publish(self)
 
         TraceDigest.new(
           span_id: span_id,
@@ -399,12 +398,14 @@ module Datadog
         attr_reader \
           :span_before_start,
           :span_finished,
-          :trace_finished
+          :trace_finished,
+          :trace_propagated
 
         def initialize
           @span_before_start = SpanBeforeStart.new
           @span_finished = SpanFinished.new
           @trace_finished = TraceFinished.new
+          @trace_propagated = TracePropagated.new
         end
 
         # Triggered before a span starts.
@@ -421,6 +422,13 @@ module Datadog
           end
         end
 
+        #  Triggered when trace is being propagated between applications or contexts
+        class TracePropagated < Tracing::Event
+          def initialize
+            super(:trace_propagated)
+          end
+        end
+
         # Triggered when the trace finishes, regardless of error.
         class TraceFinished < Tracing::Event
           def initialize