ddtrace 0.54.2 → 1.18.0

data/lib/datadog/opentracer/tracer.rb

@@ -0,0 +1,212 @@
+require 'time'
+
+require_relative '../tracing/context'
+require_relative '../tracing/tracer'
+
+module Datadog
+  module OpenTracer
+    # OpenTracing adapter for Datadog::Tracer
+    # @public_api
+    class Tracer < ::OpenTracing::Tracer
+      # (see Datadog::Tracer)
+      # @return [Datadog::Tracer]
+      attr_reader \
+        :datadog_tracer
+
+      # (see Datadog::Tracer#initialize)
+      def initialize(**options)
+        super()
+        @datadog_tracer = Datadog::Tracing::Tracer.new(**options)
+      end
+
+      # @return [ScopeManager] the current ScopeManager.
+      def scope_manager
+        @scope_manager ||= ThreadLocalScopeManager.new
+      end
+
+      # Returns a newly started and activated {Scope}.
+      #
+      # If `scope_manager.active` is not nil, no explicit references
+      # are provided, and `ignore_active_scope` is false, then an inferred
+      # {https://www.rubydoc.info/gems/opentracing/0.5.0/OpenTracing/Reference OpenTracing::Reference#CHILD_OF}
+      # reference is created to the `scope_manager.active`'s
+      # {SpanContext} when {#start_active_span} is invoked.
+      #
+      # @param operation_name [String] The operation name for the Span
+      # @param child_of [SpanContext, Span] SpanContext that acts as a parent to
+      #   the newly-started Span. If a Span instance is provided, its
+      #   context is automatically substituted. See [OpenTracing::Reference] for more
+      #   information.
+      #   If specified, the `references` parameter must be omitted.
+      # @param references [Array<OpenTracing::Reference>] An array of reference
+      #   objects that identify one or more parent SpanContexts.
+      # @param start_time [Time] When the Span started, if not now
+      # @param tags [Hash] Tags to assign to the Span at start time
+      # @param ignore_active_scope [Boolean] whether to create an implicit
+      #   OpenTracing::Reference#CHILD_OF reference to the ScopeManager#active.
+      # @param finish_on_close [Boolean] whether span should automatically be
+      #   finished when Scope#close is called
+      # @yield [Scope] If an optional block is passed to start_active it will
+      #   yield the newly-started Scope. If `finish_on_close` is true then the
+      #   Span will be finished automatically after the block is executed.
+      # @return [Scope] The newly-started and activated Scope
+      def start_active_span(
+        operation_name,
+        child_of: nil,
+        references: nil,
+        start_time: Time.now,
+        tags: nil,
+        ignore_active_scope: false,
+        finish_on_close: true
+      )
+
+        # When meant to automatically determine the parent,
+        # Use the active scope first, otherwise fall back to any
+        # context generated by Datadog, so as to append to it and gain
+        # the benefit of any out-of-the-box tracing from Datadog preceding
+        # the OpenTracer::Tracer.
+        #
+        # We do this here instead of in #start_span because #start_span generates
+        # spans that are not assigned to a scope, a.k.a not supposed to be used by
+        # subsequent spans implicitly. By using the existing Datadog context, the span
+        # effectively ends up "assigned to a scope", by virtue of being added to the
+        # Context. Hence, it would behave more like an active span, which is why it
+        # should only be here.
+        unless child_of || ignore_active_scope
+          child_of = if scope_manager.active
+                       scope_manager.active.span.context
+                     else
+                       SpanContextFactory.build(datadog_context: datadog_tracer.send(:call_context))
+                     end
+        end
+
+        # Create the span, and auto-add it to the Datadog context.
+        span = start_span(
+          operation_name,
+          child_of: child_of,
+          references: references,
+          start_time: start_time,
+          tags: tags,
+          ignore_active_scope: ignore_active_scope
+        )
+
+        # Overwrite the tracer context with the OpenTracing managed context.
+        # This is mostly for the benefit of any out-of-the-box tracing from Datadog,
+        # such that spans generated by that tracing will be attached to the OpenTracer
+        # parent span.
+        datadog_tracer.provider.context = span.context.datadog_context
+
+        scope_manager.activate(span, finish_on_close: finish_on_close).tap do |scope|
+          if block_given?
+            begin
+              yield(scope)
+            ensure
+              scope.close
+            end
+          end
+        end
+      end
+
+      # Like {#start_active_span}, but the returned {Span} has not been registered via the
+      # {ScopeManager}.
+      #
+      # @param operation_name [String] The operation name for the Span
+      # @param child_of [SpanContext, Span] SpanContext that acts as a parent to
+      #   the newly-started Span. If a Span instance is provided, its
+      #   context is automatically substituted. See [Reference] for more
+      #   information.
+      #   If specified, the `references` parameter must be omitted.
+      # @param references [Array<Reference>] An array of reference
+      #   objects that identify one or more parent SpanContexts.
+      # @param start_time [Time] When the Span started, if not now
+      # @param tags [Hash] Tags to assign to the Span at start time
+      # @param ignore_active_scope [Boolean] whether to create an implicit
+      #   References#CHILD_OF reference to the ScopeManager#active.
+      # @return [Span] the newly-started Span instance, which has not been
+      #   automatically registered via the ScopeManager
+      def start_span(
+        operation_name,
+        child_of: nil,
+        references: nil,
+        start_time: Time.now,
+        tags: nil,
+        ignore_active_scope: false
+      )
+
+        # Derive the OpenTracer::SpanContext to inherit from.
+        parent_span_context = inherited_span_context(child_of, ignore_active_scope: ignore_active_scope)
+
+        # Retrieve Datadog::Context from parent SpanContext.
+        datadog_context = parent_span_context.nil? ? Datadog::Tracing::Context.new : parent_span_context.datadog_context
+        datadog_trace_digest = parent_span_context && parent_span_context.datadog_trace_digest
+
+        # Build the new Datadog span
+        datadog_span = datadog_tracer.trace(
+          operation_name,
+          continue_from: datadog_trace_digest,
+          _context: datadog_context,
+          start_time: start_time,
+          tags: tags || {}
+        )
+
+        span_context = SpanContextFactory.build(
+          datadog_context: datadog_context || datadog_tracer.send(:call_context),
+          baggage: parent_span_context ? parent_span_context.baggage.dup : {}
+        )
+
+        # Wrap the Datadog span and OpenTracer::Span context in a OpenTracer::Span
+        Span.new(datadog_span: datadog_span, span_context: span_context)
+      end
+
+      # Inject a {SpanContext} into the given carrier.
+      #
+      # @param span_context [SpanContext]
+      # @param format [OpenTracing::FORMAT_TEXT_MAP, OpenTracing::FORMAT_BINARY, OpenTracing::FORMAT_RACK]
+      # @param carrier [Carrier] A carrier object of the type dictated by the specified `format`
+      def inject(span_context, format, carrier)
+        case format
+        when OpenTracing::FORMAT_TEXT_MAP
+          TextMapPropagator.inject(span_context, carrier)
+        when OpenTracing::FORMAT_BINARY
+          BinaryPropagator.inject(span_context, carrier)
+        when OpenTracing::FORMAT_RACK
+          RackPropagator.inject(span_context, carrier)
+        else
+          warn 'Unknown inject format'
+        end
+      end
+
+      # Extract a {SpanContext} in the given format from the given carrier.
+      #
+      # @param format [OpenTracing::FORMAT_TEXT_MAP, OpenTracing::FORMAT_BINARY, OpenTracing::FORMAT_RACK]
+      # @param carrier [Carrier] A carrier object of the type dictated by the specified `format`
+      # @return [SpanContext, nil] the extracted SpanContext or nil if none could be found
+      def extract(format, carrier)
+        case format
+        when OpenTracing::FORMAT_TEXT_MAP
+          TextMapPropagator.extract(carrier)
+        when OpenTracing::FORMAT_BINARY
+          BinaryPropagator.extract(carrier)
+        when OpenTracing::FORMAT_RACK
+          RackPropagator.extract(carrier)
+        else
+          warn 'Unknown extract format'
+          nil
+        end
+      end
+
+      private
+
+      def inherited_span_context(parent, ignore_active_scope: false)
+        case parent
+        when Span
+          parent.context
+        when SpanContext
+          parent
+        else
+          ignore_active_scope ? nil : scope_manager.active && scope_manager.active.span.context
+        end
+      end
+    end
+  end
+end

data/lib/datadog/opentracer.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require 'opentracing'
+require 'opentracing/carrier'
+require_relative 'tracing'
+
+require_relative 'opentracer/carrier'
+require_relative 'opentracer/tracer'
+require_relative 'opentracer/span'
+require_relative 'opentracer/span_context'
+require_relative 'opentracer/span_context_factory'
+require_relative 'opentracer/scope'
+require_relative 'opentracer/scope_manager'
+require_relative 'opentracer/thread_local_scope'
+require_relative 'opentracer/thread_local_scope_manager'
+require_relative 'opentracer/distributed_headers'
+require_relative 'opentracer/propagator'
+require_relative 'opentracer/text_map_propagator'
+require_relative 'opentracer/binary_propagator'
+require_relative 'opentracer/rack_propagator'
+require_relative 'opentracer/global_tracer'
+
+# Modify the OpenTracing module functions
+::OpenTracing.singleton_class.prepend(Datadog::OpenTracer::GlobalTracer)
+
+module Datadog
+  # Datadog OpenTracing integration.
+  # DEV: This module should be named `Datadog::OpenTracing` to match the gem (`opentracing`).
+  module OpenTracer
+    # Used by Telemetry to decide if OpenTracing instrumentation is enabled
+    LOADED = true
+  end
+end

data/lib/datadog/profiling/collectors/code_provenance.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+require 'set'
+require 'json'
+
+module Datadog
+  module Profiling
+    module Collectors
+      # Collects library metadata for loaded files ($LOADED_FEATURES) in the Ruby VM.
+      # The output of this class is a list of libraries which have been require'd (in particular, this is
+      # not a list of ALL installed libraries).
+      #
+      # This metadata powers grouping and categorization of stack trace data.
+      #
+      # This class acts both as a collector (collecting data) as well as a recorder (records/serializes it)
+      class CodeProvenance
+        def initialize(standard_library_path: RbConfig::CONFIG.fetch('rubylibdir'))
+          @libraries_by_name = {}
+          @libraries_by_path = {}
+          @seen_files = Set.new
+          @seen_libraries = Set.new
+
+          record_library(
+            Library.new(
+              kind: 'standard library',
+              name: 'stdlib',
+              version: RUBY_VERSION,
+              path: standard_library_path,
+            )
+          )
+        end
+
+        def refresh(loaded_files: $LOADED_FEATURES, loaded_specs: Gem.loaded_specs.values)
+          record_loaded_specs(loaded_specs)
+          record_loaded_files(loaded_files)
+
+          self
+        end
+
+        def generate
+          seen_libraries
+        end
+
+        def generate_json
+          JSON.fast_generate(v1: seen_libraries.to_a)
+        end
+
+        private
+
+        attr_reader \
+          :libraries_by_name,
+          :libraries_by_path,
+          :seen_files,
+          :seen_libraries
+
+        def record_library(library)
+          libraries_by_name[library.name] = library
+          libraries_by_path[library.path] = library
+        end
+
+        # Ruby hash maps are guaranteed to keep the insertion order of keys. Here, we sort @libraries_by_path so
+        # that the hash can be iterated in reverse order of paths.
+        #
+        # Why we do this: We do this to make sure that if there are libraries with paths that are prefixes of other
+        # libraries, e.g. '/home/foo' and '/home/foo/bar', we match to the longest path first.
+        # When reverse sorting paths as strings, '/home/foo/bar' will come before '/home/foo'.
+        #
+        # This way, when we iterate the @libraries_by_path hash, we know the first hit will also be the longest.
+        #
+        # Alternatively/in the future we could instead use a trie to match paths, but I doubt for the data sizes we're
+        # looking at that a trie is that much faster than using Ruby's built-in native collections.
+        def sort_libraries_by_longest_path_first
+          @libraries_by_path = @libraries_by_path.sort.reverse!.to_h
+        end
+
+        def record_loaded_specs(loaded_specs)
+          recorded_library = false
+
+          loaded_specs.each do |spec|
+            next if libraries_by_name.key?(spec.name)
+
+            record_library(Library.new(kind: 'library', name: spec.name, version: spec.version, path: spec.gem_dir))
+            recorded_library = true
+          end
+
+          sort_libraries_by_longest_path_first if recorded_library
+        end
+
+        def record_loaded_files(loaded_files)
+          loaded_files.each do |file_path|
+            next if seen_files.include?(file_path)
+
+            seen_files << file_path
+
+            _, found_library = libraries_by_path.find { |library_path, _| file_path.start_with?(library_path) }
+            seen_libraries << found_library if found_library
+          end
+        end
+
+        Library = Struct.new(:kind, :name, :version, :path) do
+          def initialize(kind:, name:, version:, path:)
+            super(kind.freeze, name.dup.freeze, version.to_s.dup.freeze, path.dup.freeze)
+            freeze
+          end
+
+          def to_json(*args)
+            { kind: kind, name: name, version: version, paths: [path] }.to_json(*args)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/profiling/collectors/cpu_and_wall_time_worker.rb

@@ -0,0 +1,113 @@
+module Datadog
+  module Profiling
+    module Collectors
+      # Used to trigger the periodic execution of Collectors::ThreadState, which implements all of the sampling logic
+      # itself; this class only implements the "when to do it" part.
+      # Almost all of this class is implemented as native code.
+      #
+      # Methods prefixed with _native_ are implemented in `collectors_cpu_and_wall_time_worker.c`
+      class CpuAndWallTimeWorker
+        private
+
+        attr_accessor :failure_exception
+
+        public
+
+        def initialize(
+          gc_profiling_enabled:,
+          allocation_counting_enabled:,
+          no_signals_workaround_enabled:,
+          thread_context_collector:,
+          idle_sampling_helper: IdleSamplingHelper.new,
+          # **NOTE**: This should only be used for testing; disabling the dynamic sampling rate will increase the
+          # profiler overhead!
+          dynamic_sampling_rate_enabled: true,
+          allocation_sample_every: 0 # Currently only for testing; Setting this to > 0 can add a lot of overhead!
+        )
+          unless dynamic_sampling_rate_enabled
+            Datadog.logger.warn(
+              'Profiling dynamic sampling rate disabled. This should only be used for testing, and will increase overhead!'
+            )
+          end
+
+          if allocation_counting_enabled && allocation_sample_every > 0
+            Datadog.logger.warn(
+              "Enabled experimental allocation profiling: allocation_sample_every=#{allocation_sample_every}. This is " \
+              'experimental, not recommended, and will increase overhead!'
+            )
+          end
+
+          self.class._native_initialize(
+            self,
+            thread_context_collector,
+            gc_profiling_enabled,
+            idle_sampling_helper,
+            allocation_counting_enabled,
+            no_signals_workaround_enabled,
+            dynamic_sampling_rate_enabled,
+            allocation_sample_every,
+          )
+          @worker_thread = nil
+          @failure_exception = nil
+          @start_stop_mutex = Mutex.new
+          @idle_sampling_helper = idle_sampling_helper
+        end
+
+        def start(on_failure_proc: nil)
+          @start_stop_mutex.synchronize do
+            return if @worker_thread && @worker_thread.alive?
+
+            Datadog.logger.debug { "Starting thread for: #{self}" }
+
+            @idle_sampling_helper.start
+
+            @worker_thread = Thread.new do
+              begin
+                Thread.current.name = self.class.name
+
+                self.class._native_sampling_loop(self)
+
+                Datadog.logger.debug('CpuAndWallTimeWorker thread stopping cleanly')
+              rescue Exception => e # rubocop:disable Lint/RescueException
+                @failure_exception = e
+                Datadog.logger.warn(
+                  'CpuAndWallTimeWorker thread error. ' \
+                  "Cause: #{e.class.name} #{e.message} Location: #{Array(e.backtrace).first}"
+                )
+                on_failure_proc&.call
+              end
+            end
+            @worker_thread.name = self.class.name # Repeated from above to make sure thread gets named asap
+            @worker_thread.thread_variable_set(:fork_safe, true)
+          end
+
+          true
+        end
+
+        def stop
+          @start_stop_mutex.synchronize do
+            Datadog.logger.debug('Requesting CpuAndWallTimeWorker thread shut down')
+
+            @idle_sampling_helper.stop
+
+            return unless @worker_thread
+
+            self.class._native_stop(self, @worker_thread)
+
+            @worker_thread.join
+            @worker_thread = nil
+            @failure_exception = nil
+          end
+        end
+
+        def reset_after_fork
+          self.class._native_reset_after_fork(self)
+        end
+
+        def stats
+          self.class._native_stats(self)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/profiling/collectors/dynamic_sampling_rate.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Profiling
+    module Collectors
+      # Used to pace the rate of profiling samples based on the last observed time for a sample.
+      # All of this module is implemented as native code.
+      #
+      # Methods prefixed with _native_ are implemented in `collectors_dynamic_sampling_rate.c`
+      module DynamicSamplingRate
+      end
+    end
+  end
+end

data/lib/datadog/profiling/collectors/idle_sampling_helper.rb

@@ -0,0 +1,68 @@
+module Datadog
+  module Profiling
+    module Collectors
+      # Used by the Collectors::CpuAndWallTimeWorker to gather samples when the Ruby process is idle.
+      # Almost all of this class is implemented as native code.
+      #
+      # Methods prefixed with _native_ are implemented in `collectors_idle_sampling_helper.c`
+      class IdleSamplingHelper
+        private
+
+        attr_accessor :failure_exception
+
+        public
+
+        def initialize
+          @worker_thread = nil
+          @start_stop_mutex = Mutex.new
+        end
+
+        def start
+          @start_stop_mutex.synchronize do
+            return if @worker_thread && @worker_thread.alive?
+
+            Datadog.logger.debug { "Starting thread for: #{self}" }
+
+            # The same instance of the IdleSamplingHelper can be reused multiple times, and this resets it back to
+            # a pristine state before recreating the worker thread
+            self.class._native_reset(self)
+
+            @worker_thread = Thread.new do
+              begin
+                Thread.current.name = self.class.name
+
+                self.class._native_idle_sampling_loop(self)
+
+                Datadog.logger.debug('IdleSamplingHelper thread stopping cleanly')
+              rescue Exception => e # rubocop:disable Lint/RescueException
+                @failure_exception = e
+                Datadog.logger.warn(
+                  'IdleSamplingHelper thread error. ' \
+                  "Cause: #{e.class.name} #{e.message} Location: #{Array(e.backtrace).first}"
+                )
+              end
+            end
+            @worker_thread.name = self.class.name # Repeated from above to make sure thread gets named asap
+            @worker_thread.thread_variable_set(:fork_safe, true)
+          end
+
+          true
+        end
+
+        def stop
+          @start_stop_mutex.synchronize do
+            Datadog.logger.debug('Requesting IdleSamplingHelper thread shut down')
+
+            return unless @worker_thread
+
+            self.class._native_stop(self)
+
+            @worker_thread.join
+            @worker_thread = nil
+            @failure_exception = nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/profiling/collectors/stack.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Profiling
+    module Collectors
+      # Used to gather a stack trace from a given Ruby thread. Stores its output on a `StackRecorder`.
+      #
+      # This class is not empty; all of this class is implemented as native code.
+      class Stack # rubocop:disable Lint/EmptyClass
+      end
+    end
+  end
+end

data/lib/datadog/profiling/collectors/thread_context.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Profiling
+    module Collectors
+      # Used to trigger sampling of threads, based on external "events", such as:
+      # * periodic timer for cpu-time and wall-time
+      # * VM garbage collection events
+      # * VM object allocation events
+      # Triggering of this component (e.g. watching for the above "events") is implemented by
+      # Collectors::CpuAndWallTimeWorker.
+      # The stack collection itself is handled using the Datadog::Profiling::Collectors::Stack.
+      # Almost all of this class is implemented as native code.
+      #
+      # Methods prefixed with _native_ are implemented in `collectors_thread_context.c`
+      class ThreadContext
+        def initialize(
+          recorder:,
+          max_frames:,
+          tracer:,
+          endpoint_collection_enabled:,
+          timeline_enabled:,
+          allocation_type_enabled: true
+        )
+          tracer_context_key = safely_extract_context_key_from(tracer)
+          self.class._native_initialize(
+            self,
+            recorder,
+            max_frames,
+            tracer_context_key,
+            endpoint_collection_enabled,
+            timeline_enabled,
+            allocation_type_enabled,
+          )
+        end
+
+        def inspect
+          # Compose Ruby's default inspect with our custom inspect for the native parts
+          result = super()
+          result[-1] = "#{self.class._native_inspect(self)}>"
+          result
+        end
+
+        def reset_after_fork
+          self.class._native_reset_after_fork(self)
+        end
+
+        private
+
+        def safely_extract_context_key_from(tracer)
+          provider = tracer && tracer.respond_to?(:provider) && tracer.provider
+
+          return unless provider
+
+          context = provider.instance_variable_get(:@context)
+          context && context.instance_variable_get(:@key)
+        end
+      end
+    end
+  end
+end