ddtrace 0.34.1 → 1.23.1

data/lib/datadog/opentracer/text_map_propagator.rb

@@ -0,0 +1,87 @@
+require_relative '../tracing/context'
+require_relative '../tracing/distributed/datadog'
+require_relative '../tracing/trace_operation'
+require_relative 'propagator'
+
+module Datadog
+  module OpenTracer
+    # OpenTracing propagator for Datadog::OpenTracer::Tracer
+    module TextMapPropagator
+      extend Propagator
+
+      BAGGAGE_PREFIX = 'ot-baggage-'.freeze
+
+      class << self
+        # Inject a SpanContext into the given carrier
+        #
+        # @param span_context [SpanContext]
+        # @param carrier [Carrier] A carrier object of Rack type
+        def inject(span_context, carrier)
+          # Inject baggage
+          span_context.baggage.each do |key, value|
+            carrier[BAGGAGE_PREFIX + key] = value
+          end
+
+          # Inject Datadog trace properties
+          digest = if span_context.datadog_context && span_context.datadog_context.active_trace
+                     span_context.datadog_context.active_trace.to_digest
+                   else
+                     span_context.datadog_trace_digest
+                   end
+          return unless digest
+
+          carrier[Tracing::Distributed::Datadog::ORIGIN_KEY] = digest.trace_origin
+          carrier[Tracing::Distributed::Datadog::PARENT_ID_KEY] = digest.span_id
+          carrier[Tracing::Distributed::Datadog::SAMPLING_PRIORITY_KEY] = digest.trace_sampling_priority
+          carrier[Tracing::Distributed::Datadog::TRACE_ID_KEY] =
+            Datadog::Tracing::Utils::TraceId.to_low_order(digest.trace_id)
+
+          nil
+        end
+
+        # Extract a SpanContext in TextMap format from the given carrier.
+        #
+        # @param carrier [Carrier] A carrier object of TextMap type
+        # @return [SpanContext, nil] the extracted SpanContext or nil if none could be found
+        def extract(carrier)
+          # First extract & build a Datadog context
+          headers = DistributedHeaders.new(carrier)
+          datadog_trace_digest = headers_to_trace_digest(headers)
+
+          # Then extract any other baggage
+          baggage = {}
+          carrier.each do |key, value|
+            baggage[item_to_baggage(key)] = value if baggage_item?(key)
+          end
+
+          SpanContextFactory.build(
+            datadog_context: nil,
+            datadog_trace_digest: datadog_trace_digest,
+            baggage: baggage
+          )
+        end
+
+        private
+
+        def headers_to_trace_digest(headers)
+          return unless headers.valid?
+
+          Datadog::Tracing::TraceDigest.new(
+            span_id: headers.parent_id,
+            trace_id: headers.trace_id,
+            trace_origin: headers.origin,
+            trace_sampling_priority: headers.sampling_priority
+          )
+        end
+
+        def baggage_item?(item)
+          item.to_s.start_with?(BAGGAGE_PREFIX)
+        end
+
+        def item_to_baggage(key)
+          key[BAGGAGE_PREFIX.length, key.length]
+        end
+      end
+    end
+  end
+end

data/lib/datadog/opentracer/thread_local_scope.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Datadog
+  module OpenTracer
+    # OpenTracing adapter for thread local scopes
+    # @public_api
+    class ThreadLocalScope < Scope
+      attr_reader \
+        :finish_on_close
+
+      def initialize(
+        manager:,
+        span:,
+        finish_on_close: true
+      )
+        super(manager: manager, span: span)
+        @finish_on_close = finish_on_close
+        @previous_scope = manager.active
+      end
+
+      # Mark the end of the active period for the current thread and Scope,
+      # updating the ScopeManager#active in the process.
+      #
+      # NOTE: Calling close more than once on a single Scope instance leads to
+      # undefined behavior.
+      def close
+        return unless equal?(manager.active)
+
+        span.finish if finish_on_close
+        manager.send(:set_scope, @previous_scope)
+      end
+    end
+  end
+end

data/lib/datadog/opentracer/thread_local_scope_manager.rb

@@ -0,0 +1,64 @@
+require_relative '../core/utils/sequence'
+
+module Datadog
+  module OpenTracer
+    # OpenTracing adapter for thread local scope management
+    # @public_api
+    class ThreadLocalScopeManager < ScopeManager
+      def initialize(*args, &block)
+        super(*args, &block)
+        @thread_key = "dd_opentracer_context_#{ThreadLocalScopeManager.next_instance_id}".to_sym
+      end
+
+      ruby2_keywords :initialize if respond_to?(:ruby2_keywords, true)
+
+      # Make a span instance active.
+      #
+      # @param span [Span] the Span that should become active
+      # @param finish_on_close [Boolean] whether the Span should automatically be
+      #   finished when Scope#close is called
+      # @return [Scope] instance to control the end of the active period for the
+      #  Span. It is a programming error to neglect to call Scope#close on the
+      #  returned instance.
+      def activate(span, finish_on_close: true)
+        ThreadLocalScope.new(
+          manager: self,
+          span: span,
+          finish_on_close: finish_on_close
+        ).tap do |scope|
+          set_scope(scope)
+        end
+      end
+
+      # @return [Scope] the currently active Scope which can be used to access the
+      # currently active Span.
+      #
+      # If there is a non-null Scope, its wrapped Span becomes an implicit parent
+      # (as Reference#CHILD_OF) of any newly-created Span at Tracer#start_active_span
+      # or Tracer#start_span time.
+      def active
+        Thread.current[@thread_key]
+      end
+
+      # Ensure two instances of {FiberLocalContext} do not conflict.
+      # We previously used {FiberLocalContext#object_id} to ensure uniqueness
+      # but the VM is allowed to reuse `object_id`, allow for the possibility that
+      # a new FiberLocalContext would be able to read an old FiberLocalContext's
+      # value.
+      UNIQUE_INSTANCE_MUTEX = Mutex.new
+      UNIQUE_INSTANCE_GENERATOR = Datadog::Core::Utils::Sequence.new
+
+      private_constant :UNIQUE_INSTANCE_MUTEX, :UNIQUE_INSTANCE_GENERATOR
+
+      def self.next_instance_id
+        UNIQUE_INSTANCE_MUTEX.synchronize { UNIQUE_INSTANCE_GENERATOR.next }
+      end
+
+      private
+
+      def set_scope(scope)
+        Thread.current[@thread_key] = scope
+      end
+    end
+  end
+end

data/lib/datadog/opentracer/tracer.rb

@@ -0,0 +1,212 @@
+require 'time'
+
+require_relative '../tracing/context'
+require_relative '../tracing/tracer'
+
+module Datadog
+  module OpenTracer
+    # OpenTracing adapter for Datadog::Tracer
+    # @public_api
+    class Tracer < ::OpenTracing::Tracer
+      # (see Datadog::Tracer)
+      # @return [Datadog::Tracer]
+      attr_reader \
+        :datadog_tracer
+
+      # (see Datadog::Tracer#initialize)
+      def initialize(**options)
+        super()
+        @datadog_tracer = Datadog::Tracing::Tracer.new(**options)
+      end
+
+      # @return [ScopeManager] the current ScopeManager.
+      def scope_manager
+        @scope_manager ||= ThreadLocalScopeManager.new
+      end
+
+      # Returns a newly started and activated {Scope}.
+      #
+      # If `scope_manager.active` is not nil, no explicit references
+      # are provided, and `ignore_active_scope` is false, then an inferred
+      # {https://www.rubydoc.info/gems/opentracing/0.5.0/OpenTracing/Reference OpenTracing::Reference#CHILD_OF}
+      # reference is created to the `scope_manager.active`'s
+      # {SpanContext} when {#start_active_span} is invoked.
+      #
+      # @param operation_name [String] The operation name for the Span
+      # @param child_of [SpanContext, Span] SpanContext that acts as a parent to
+      #   the newly-started Span. If a Span instance is provided, its
+      #   context is automatically substituted. See [OpenTracing::Reference] for more
+      #   information.
+      #   If specified, the `references` parameter must be omitted.
+      # @param references [Array<OpenTracing::Reference>] An array of reference
+      #   objects that identify one or more parent SpanContexts.
+      # @param start_time [Time] When the Span started, if not now
+      # @param tags [Hash] Tags to assign to the Span at start time
+      # @param ignore_active_scope [Boolean] whether to create an implicit
+      #   OpenTracing::Reference#CHILD_OF reference to the ScopeManager#active.
+      # @param finish_on_close [Boolean] whether span should automatically be
+      #   finished when Scope#close is called
+      # @yield [Scope] If an optional block is passed to start_active it will
+      #   yield the newly-started Scope. If `finish_on_close` is true then the
+      #   Span will be finished automatically after the block is executed.
+      # @return [Scope] The newly-started and activated Scope
+      def start_active_span(
+        operation_name,
+        child_of: nil,
+        references: nil,
+        start_time: Time.now,
+        tags: nil,
+        ignore_active_scope: false,
+        finish_on_close: true
+      )
+
+        # When meant to automatically determine the parent,
+        # Use the active scope first, otherwise fall back to any
+        # context generated by Datadog, so as to append to it and gain
+        # the benefit of any out-of-the-box tracing from Datadog preceding
+        # the OpenTracer::Tracer.
+        #
+        # We do this here instead of in #start_span because #start_span generates
+        # spans that are not assigned to a scope, a.k.a not supposed to be used by
+        # subsequent spans implicitly. By using the existing Datadog context, the span
+        # effectively ends up "assigned to a scope", by virtue of being added to the
+        # Context. Hence, it would behave more like an active span, which is why it
+        # should only be here.
+        unless child_of || ignore_active_scope
+          child_of = if scope_manager.active
+                       scope_manager.active.span.context
+                     else
+                       SpanContextFactory.build(datadog_context: datadog_tracer.send(:call_context))
+                     end
+        end
+
+        # Create the span, and auto-add it to the Datadog context.
+        span = start_span(
+          operation_name,
+          child_of: child_of,
+          references: references,
+          start_time: start_time,
+          tags: tags,
+          ignore_active_scope: ignore_active_scope
+        )
+
+        # Overwrite the tracer context with the OpenTracing managed context.
+        # This is mostly for the benefit of any out-of-the-box tracing from Datadog,
+        # such that spans generated by that tracing will be attached to the OpenTracer
+        # parent span.
+        datadog_tracer.provider.context = span.context.datadog_context
+
+        scope_manager.activate(span, finish_on_close: finish_on_close).tap do |scope|
+          if block_given?
+            begin
+              yield(scope)
+            ensure
+              scope.close
+            end
+          end
+        end
+      end
+
+      # Like {#start_active_span}, but the returned {Span} has not been registered via the
+      # {ScopeManager}.
+      #
+      # @param operation_name [String] The operation name for the Span
+      # @param child_of [SpanContext, Span] SpanContext that acts as a parent to
+      #   the newly-started Span. If a Span instance is provided, its
+      #   context is automatically substituted. See [Reference] for more
+      #   information.
+      #   If specified, the `references` parameter must be omitted.
+      # @param references [Array<Reference>] An array of reference
+      #   objects that identify one or more parent SpanContexts.
+      # @param start_time [Time] When the Span started, if not now
+      # @param tags [Hash] Tags to assign to the Span at start time
+      # @param ignore_active_scope [Boolean] whether to create an implicit
+      #   References#CHILD_OF reference to the ScopeManager#active.
+      # @return [Span] the newly-started Span instance, which has not been
+      #   automatically registered via the ScopeManager
+      def start_span(
+        operation_name,
+        child_of: nil,
+        references: nil,
+        start_time: Time.now,
+        tags: nil,
+        ignore_active_scope: false
+      )
+
+        # Derive the OpenTracer::SpanContext to inherit from.
+        parent_span_context = inherited_span_context(child_of, ignore_active_scope: ignore_active_scope)
+
+        # Retrieve Datadog::Context from parent SpanContext.
+        datadog_context = parent_span_context.nil? ? Datadog::Tracing::Context.new : parent_span_context.datadog_context
+        datadog_trace_digest = parent_span_context && parent_span_context.datadog_trace_digest
+
+        # Build the new Datadog span
+        datadog_span = datadog_tracer.trace(
+          operation_name,
+          continue_from: datadog_trace_digest,
+          _context: datadog_context,
+          start_time: start_time,
+          tags: tags || {}
+        )
+
+        span_context = SpanContextFactory.build(
+          datadog_context: datadog_context || datadog_tracer.send(:call_context),
+          baggage: parent_span_context ? parent_span_context.baggage.dup : {}
+        )
+
+        # Wrap the Datadog span and OpenTracer::Span context in a OpenTracer::Span
+        Span.new(datadog_span: datadog_span, span_context: span_context)
+      end
+
+      # Inject a {SpanContext} into the given carrier.
+      #
+      # @param span_context [SpanContext]
+      # @param format [OpenTracing::FORMAT_TEXT_MAP, OpenTracing::FORMAT_BINARY, OpenTracing::FORMAT_RACK]
+      # @param carrier [Carrier] A carrier object of the type dictated by the specified `format`
+      def inject(span_context, format, carrier)
+        case format
+        when OpenTracing::FORMAT_TEXT_MAP
+          TextMapPropagator.inject(span_context, carrier)
+        when OpenTracing::FORMAT_BINARY
+          BinaryPropagator.inject(span_context, carrier)
+        when OpenTracing::FORMAT_RACK
+          RackPropagator.inject(span_context, carrier)
+        else
+          warn 'Unknown inject format'
+        end
+      end
+
+      # Extract a {SpanContext} in the given format from the given carrier.
+      #
+      # @param format [OpenTracing::FORMAT_TEXT_MAP, OpenTracing::FORMAT_BINARY, OpenTracing::FORMAT_RACK]
+      # @param carrier [Carrier] A carrier object of the type dictated by the specified `format`
+      # @return [SpanContext, nil] the extracted SpanContext or nil if none could be found
+      def extract(format, carrier)
+        case format
+        when OpenTracing::FORMAT_TEXT_MAP
+          TextMapPropagator.extract(carrier)
+        when OpenTracing::FORMAT_BINARY
+          BinaryPropagator.extract(carrier)
+        when OpenTracing::FORMAT_RACK
+          RackPropagator.extract(carrier)
+        else
+          warn 'Unknown extract format'
+          nil
+        end
+      end
+
+      private
+
+      def inherited_span_context(parent, ignore_active_scope: false)
+        case parent
+        when Span
+          parent.context
+        when SpanContext
+          parent
+        else
+          ignore_active_scope ? nil : scope_manager.active && scope_manager.active.span.context
+        end
+      end
+    end
+  end
+end

data/lib/datadog/opentracer.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require 'opentracing'
+require 'opentracing/carrier'
+require_relative 'tracing'
+
+require_relative 'opentracer/carrier'
+require_relative 'opentracer/tracer'
+require_relative 'opentracer/span'
+require_relative 'opentracer/span_context'
+require_relative 'opentracer/span_context_factory'
+require_relative 'opentracer/scope'
+require_relative 'opentracer/scope_manager'
+require_relative 'opentracer/thread_local_scope'
+require_relative 'opentracer/thread_local_scope_manager'
+require_relative 'opentracer/distributed_headers'
+require_relative 'opentracer/propagator'
+require_relative 'opentracer/text_map_propagator'
+require_relative 'opentracer/binary_propagator'
+require_relative 'opentracer/rack_propagator'
+require_relative 'opentracer/global_tracer'
+
+# Modify the OpenTracing module functions
+::OpenTracing.singleton_class.prepend(Datadog::OpenTracer::GlobalTracer)
+
+module Datadog
+  # Datadog OpenTracing integration.
+  # DEV: This module should be named `Datadog::OpenTracing` to match the gem (`opentracing`).
+  module OpenTracer
+    # Used by Telemetry to decide if OpenTracing instrumentation is enabled
+    LOADED = true
+  end
+end

data/lib/datadog/profiling/collectors/code_provenance.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+require 'set'
+require 'json'
+
+module Datadog
+  module Profiling
+    module Collectors
+      # Collects library metadata for loaded files ($LOADED_FEATURES) in the Ruby VM.
+      # The output of this class is a list of libraries which have been require'd (in particular, this is
+      # not a list of ALL installed libraries).
+      #
+      # This metadata powers grouping and categorization of stack trace data.
+      #
+      # This class acts both as a collector (collecting data) as well as a recorder (records/serializes it)
+      class CodeProvenance
+        def initialize(standard_library_path: RbConfig::CONFIG.fetch('rubylibdir'))
+          @libraries_by_name = {}
+          @libraries_by_path = {}
+          @seen_files = Set.new
+          @seen_libraries = Set.new
+
+          record_library(
+            Library.new(
+              kind: 'standard library',
+              name: 'stdlib',
+              version: RUBY_VERSION,
+              path: standard_library_path,
+            )
+          )
+        end
+
+        def refresh(loaded_files: $LOADED_FEATURES, loaded_specs: Gem.loaded_specs.values)
+          record_loaded_specs(loaded_specs)
+          record_loaded_files(loaded_files)
+
+          self
+        end
+
+        def generate
+          seen_libraries
+        end
+
+        def generate_json
+          JSON.fast_generate(v1: seen_libraries.to_a)
+        end
+
+        private
+
+        attr_reader \
+          :libraries_by_name,
+          :libraries_by_path,
+          :seen_files,
+          :seen_libraries
+
+        def record_library(library)
+          libraries_by_name[library.name] = library
+          libraries_by_path[library.path] = library
+        end
+
+        # Ruby hash maps are guaranteed to keep the insertion order of keys. Here, we sort @libraries_by_path so
+        # that the hash can be iterated in reverse order of paths.
+        #
+        # Why we do this: We do this to make sure that if there are libraries with paths that are prefixes of other
+        # libraries, e.g. '/home/foo' and '/home/foo/bar', we match to the longest path first.
+        # When reverse sorting paths as strings, '/home/foo/bar' will come before '/home/foo'.
+        #
+        # This way, when we iterate the @libraries_by_path hash, we know the first hit will also be the longest.
+        #
+        # Alternatively/in the future we could instead use a trie to match paths, but I doubt for the data sizes we're
+        # looking at that a trie is that much faster than using Ruby's built-in native collections.
+        def sort_libraries_by_longest_path_first
+          @libraries_by_path = @libraries_by_path.sort.reverse!.to_h
+        end
+
+        def record_loaded_specs(loaded_specs)
+          recorded_library = false
+
+          loaded_specs.each do |spec|
+            next if libraries_by_name.key?(spec.name)
+
+            record_library(Library.new(kind: 'library', name: spec.name, version: spec.version, path: spec.gem_dir))
+            recorded_library = true
+          end
+
+          sort_libraries_by_longest_path_first if recorded_library
+        end
+
+        def record_loaded_files(loaded_files)
+          loaded_files.each do |file_path|
+            next if seen_files.include?(file_path)
+
+            seen_files << file_path
+
+            _, found_library = libraries_by_path.find { |library_path, _| file_path.start_with?(library_path) }
+            seen_libraries << found_library if found_library
+          end
+        end
+
+        # Represents metadata we have for a ruby gem
+        class Library
+          attr_reader :kind, :name, :version, :path
+
+          def initialize(kind:, name:, version:, path:)
+            @kind = kind.freeze
+            @name = name.dup.freeze
+            @version = version.to_s.dup.freeze
+            @path = path.dup.freeze
+            freeze
+          end
+
+          def to_json(arg = nil)
+            { kind: @kind, name: @name, version: @version, paths: [@path] }.to_json(arg)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/profiling/collectors/cpu_and_wall_time_worker.rb

@@ -0,0 +1,137 @@
+module Datadog
+  module Profiling
+    module Collectors
+      # Used to trigger the periodic execution of Collectors::ThreadState, which implements all of the sampling logic
+      # itself; this class only implements the "when to do it" part.
+      # Almost all of this class is implemented as native code.
+      #
+      # Methods prefixed with _native_ are implemented in `collectors_cpu_and_wall_time_worker.c`
+      class CpuAndWallTimeWorker
+        private
+
+        attr_accessor :failure_exception
+
+        public
+
+        def initialize(
+          gc_profiling_enabled:,
+          no_signals_workaround_enabled:,
+          thread_context_collector:,
+          dynamic_sampling_rate_overhead_target_percentage:,
+          allocation_profiling_enabled:,
+          # **NOTE**: This should only be used for testing; disabling the dynamic sampling rate will increase the
+          # profiler overhead!
+          dynamic_sampling_rate_enabled: true,
+          skip_idle_samples_for_testing: false,
+          idle_sampling_helper: IdleSamplingHelper.new
+        )
+          unless dynamic_sampling_rate_enabled
+            Datadog.logger.warn(
+              'Profiling dynamic sampling rate disabled. This should only be used for testing, and will increase overhead!'
+            )
+          end
+
+          self.class._native_initialize(
+            self,
+            thread_context_collector,
+            gc_profiling_enabled,
+            idle_sampling_helper,
+            no_signals_workaround_enabled,
+            dynamic_sampling_rate_enabled,
+            dynamic_sampling_rate_overhead_target_percentage,
+            allocation_profiling_enabled,
+            skip_idle_samples_for_testing,
+          )
+          @worker_thread = nil
+          @failure_exception = nil
+          @start_stop_mutex = Mutex.new
+          @idle_sampling_helper = idle_sampling_helper
+          @wait_until_running_mutex = Mutex.new
+          @wait_until_running_condition = ConditionVariable.new
+        end
+
+        def start(on_failure_proc: nil)
+          @start_stop_mutex.synchronize do
+            return if @worker_thread && @worker_thread.alive?
+
+            Datadog.logger.debug { "Starting thread for: #{self}" }
+
+            @idle_sampling_helper.start
+
+            @worker_thread = Thread.new do
+              begin
+                Thread.current.name = self.class.name
+
+                self.class._native_sampling_loop(self)
+
+                Datadog.logger.debug('CpuAndWallTimeWorker thread stopping cleanly')
+              rescue Exception => e # rubocop:disable Lint/RescueException
+                @failure_exception = e
+                Datadog.logger.warn(
+                  'CpuAndWallTimeWorker thread error. ' \
+                  "Cause: #{e.class.name} #{e.message} Location: #{Array(e.backtrace).first}"
+                )
+                on_failure_proc&.call
+              end
+            end
+            @worker_thread.name = self.class.name # Repeated from above to make sure thread gets named asap
+            @worker_thread.thread_variable_set(:fork_safe, true)
+          end
+
+          true
+        end
+
+        def stop
+          @start_stop_mutex.synchronize do
+            Datadog.logger.debug('Requesting CpuAndWallTimeWorker thread shut down')
+
+            @idle_sampling_helper.stop
+
+            return unless @worker_thread
+
+            self.class._native_stop(self, @worker_thread)
+
+            @worker_thread.join
+            @worker_thread = nil
+            @failure_exception = nil
+          end
+        end
+
+        def reset_after_fork
+          self.class._native_reset_after_fork(self)
+        end
+
+        def stats
+          self.class._native_stats(self)
+        end
+
+        def stats_and_reset_not_thread_safe
+          stats = self.stats
+          self.class._native_stats_reset_not_thread_safe(self)
+          stats
+        end
+
+        # Useful for testing, to e.g. make sure the profiler is running before we start running some code we want to observe
+        def wait_until_running(timeout_seconds: 5)
+          @wait_until_running_mutex.synchronize do
+            return true if self.class._native_is_running?(self)
+
+            @wait_until_running_condition.wait(@wait_until_running_mutex, timeout_seconds)
+
+            if self.class._native_is_running?(self)
+              true
+            else
+              raise "Timeout waiting for #{self.class.name} to start (waited for #{timeout_seconds} seconds)"
+            end
+          end
+        end
+
+        private
+
+        def signal_running
+          @wait_until_running_mutex.synchronize { @wait_until_running_condition.broadcast }
+        end
+      end
+    end
+  end
+end