ddtrace 0.54.2 → 1.2.0

data/lib/datadog/profiling/flush.rb

@@ -0,0 +1,41 @@
+# typed: false
+
+require 'datadog/core/environment/identity'
+require 'datadog/core/environment/socket'
+
+module Datadog
+  module Profiling
+    # Represents a collection of events of a specific type being flushed.
+    EventGroup = Struct.new(:event_class, :events)
+
+    # Entity class used to represent metadata for a given profile
+    class Flush
+      attr_reader \
+        :start,
+        :finish,
+        :pprof_file_name,
+        :pprof_data, # gzipped pprof bytes
+        :code_provenance_file_name,
+        :code_provenance_data, # gzipped json bytes
+        :tags_as_array
+
+      def initialize(
+        start:,
+        finish:,
+        pprof_file_name:,
+        pprof_data:,
+        code_provenance_file_name:,
+        code_provenance_data:,
+        tags_as_array:
+      )
+        @start = start
+        @finish = finish
+        @pprof_file_name = pprof_file_name
+        @pprof_data = pprof_data
+        @code_provenance_file_name = code_provenance_file_name
+        @code_provenance_data = code_provenance_data
+        @tags_as_array = tags_as_array
+      end
+    end
+  end
+end

data/lib/datadog/profiling/http_transport.rb

@@ -0,0 +1,131 @@
+# typed: false
+
+module Datadog
+  module Profiling
+    # Used to report profiling data to Datadog.
+    # Methods prefixed with _native_ are implemented in `http_transport.c`
+    class HttpTransport
+      def initialize(agent_settings:, site:, api_key:, upload_timeout_seconds:)
+        @upload_timeout_milliseconds = (upload_timeout_seconds * 1_000).to_i
+
+        validate_agent_settings(agent_settings)
+
+        @exporter_configuration =
+          if agentless?(site, api_key)
+            [:agentless, site, api_key]
+          else
+            [:agent, base_url_from(agent_settings)]
+          end
+
+        status, result = validate_exporter(@exporter_configuration)
+
+        raise(ArgumentError, "Failed to initialize transport: #{result}") if status == :error
+      end
+
+      def export(flush)
+        status, result = do_export(
+          exporter_configuration: @exporter_configuration,
+          upload_timeout_milliseconds: @upload_timeout_milliseconds,
+
+          # why "timespec"?
+          # libddprof represents time using POSIX's struct timespec, see
+          # https://www.gnu.org/software/libc/manual/html_node/Time-Types.html
+          # aka it represents the seconds part separate from the nanoseconds part
+          start_timespec_seconds: flush.start.tv_sec,
+          start_timespec_nanoseconds: flush.start.tv_nsec,
+          finish_timespec_seconds: flush.finish.tv_sec,
+          finish_timespec_nanoseconds: flush.finish.tv_nsec,
+
+          pprof_file_name: flush.pprof_file_name,
+          pprof_data: flush.pprof_data,
+          code_provenance_file_name: flush.code_provenance_file_name,
+          code_provenance_data: flush.code_provenance_data,
+
+          tags_as_array: flush.tags_as_array,
+        )
+
+        if status == :ok
+          if (200..299).cover?(result)
+            Datadog.logger.debug('Successfully reported profiling data')
+            true
+          else
+            Datadog.logger.error("Failed to report profiling data: server returned unexpected HTTP #{result} status code")
+            false
+          end
+        else
+          Datadog.logger.error("Failed to report profiling data: #{result}")
+          false
+        end
+      end
+
+      # Used to log soft failures in `ddprof_ffi_Vec_tag_push` (e.g. we still report the profile in these cases)
+      # Called from native code
+      def self.log_failure_to_process_tag(failure_details)
+        Datadog.logger.warn("Failed to add tag to profiling request: #{failure_details}")
+      end
+
+      private
+
+      def base_url_from(agent_settings)
+        case agent_settings.adapter
+        when Datadog::Transport::Ext::HTTP::ADAPTER
+          "#{agent_settings.ssl ? 'https' : 'http'}://#{agent_settings.hostname}:#{agent_settings.port}/"
+        when Datadog::Transport::Ext::UnixSocket::ADAPTER
+          "unix://#{agent_settings.uds_path}"
+        else
+          raise ArgumentError, "Unexpected adapter: #{agent_settings.adapter}"
+        end
+      end
+
+      def validate_agent_settings(agent_settings)
+        supported_adapters = [Datadog::Transport::Ext::HTTP::ADAPTER, Datadog::Transport::Ext::UnixSocket::ADAPTER]
+        unless supported_adapters.include?(agent_settings.adapter)
+          raise ArgumentError, "Unsupported transport configuration for profiling: Adapter #{agent_settings.adapter} " \
+            ' is not supported'
+        end
+
+        if agent_settings.deprecated_for_removal_transport_configuration_proc
+          Datadog.logger.warn(
+            'Ignoring custom c.tracing.transport_options setting as the profiler does not support it.'
+          )
+        end
+      end
+
+      def agentless?(site, api_key)
+        site && api_key && Core::Environment::VariableHelpers.env_to_bool(Profiling::Ext::ENV_AGENTLESS, false)
+      end
+
+      def validate_exporter(exporter_configuration)
+        self.class._native_validate_exporter(exporter_configuration)
+      end
+
+      def do_export(
+        exporter_configuration:,
+        upload_timeout_milliseconds:,
+        start_timespec_seconds:,
+        start_timespec_nanoseconds:,
+        finish_timespec_seconds:,
+        finish_timespec_nanoseconds:,
+        pprof_file_name:,
+        pprof_data:,
+        code_provenance_file_name:,
+        code_provenance_data:,
+        tags_as_array:
+      )
+        self.class._native_do_export(
+          exporter_configuration,
+          upload_timeout_milliseconds,
+          start_timespec_seconds,
+          start_timespec_nanoseconds,
+          finish_timespec_seconds,
+          finish_timespec_nanoseconds,
+          pprof_file_name,
+          pprof_data,
+          code_provenance_file_name,
+          code_provenance_data,
+          tags_as_array,
+        )
+      end
+    end
+  end
+end

data/lib/datadog/profiling/load_native_extension.rb

@@ -0,0 +1,22 @@
+# typed: ignore
+
+# This file is used to load the profiling native extension. It works in two steps:
+#
+# 1. Load the ddtrace_profiling_loader extension. This extension will be used to load the actual extension, but in
+#    a special way that avoids exposing native-level code symbols. See `ddtrace_profiling_loader.c` for more details.
+#
+# 2. Use the Datadog::Profiling::Loader exposed by the ddtrace_profiling_loader extension to load the actual
+#    profiling native extension.
+#
+# All code on this file is on-purpose at the top-level; this makes it so this file is executed only once,
+# the first time it gets required, to avoid any issues with the native extension being initialized more than once.
+
+require "ddtrace_profiling_loader.#{RUBY_VERSION}_#{RUBY_PLATFORM}"
+
+extension_name = "ddtrace_profiling_native_extension.#{RUBY_VERSION}_#{RUBY_PLATFORM}"
+full_file_path = "#{__dir__}/../../#{extension_name}.#{RbConfig::CONFIG['DLEXT']}"
+init_function_name = "Init_#{extension_name.split('.').first}"
+
+status, result = Datadog::Profiling::Loader._native_load(full_file_path, init_function_name)
+
+raise "Failure to load #{extension_name} due to #{result}" if status == :error

data/lib/datadog/profiling/native_extension.rb

@@ -0,0 +1,41 @@
+# typed: true
+
+module Datadog
+  module Profiling
+    # This module contains classes and methods which are implemented using native code in the
+    # ext/ddtrace_profiling_native_extension folder, as well as some Ruby-level utilities that don't make sense to
+    # write using C
+    module NativeExtension
+      private_class_method def self.working?
+        native_working?
+      end
+
+      unless singleton_class.private_method_defined?(:native_working?)
+        private_class_method def self.native_working?
+          false
+        end
+      end
+
+      unless singleton_class.method_defined?(:clock_id_for)
+        def self.clock_id_for(_)
+          nil
+        end
+      end
+
+      def self.cpu_time_ns_for(thread)
+        clock_id =
+          begin
+            clock_id_for(thread)
+          rescue Errno::ESRCH
+            nil
+          end
+
+        begin
+          ::Process.clock_gettime(clock_id, :nanosecond) if clock_id
+        rescue Errno::EINVAL
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/datadog/profiling/old_ext.rb

@@ -0,0 +1,42 @@
+# typed: true
+
+module Datadog
+  module Profiling
+    # NOTE: This OldExt file is temporary and expected to be removed once the migration to the new `HttpTransport` class
+    # is complete
+    module OldExt
+      module Transport
+        module HTTP
+          URI_TEMPLATE_DD_API = 'https://intake.profile.%s/'.freeze
+
+          FORM_FIELD_RECORDING_START = 'start'.freeze
+          FORM_FIELD_RECORDING_END = 'end'.freeze
+          FORM_FIELD_FAMILY = 'family'.freeze
+          FORM_FIELD_TAG_ENV = 'env'.freeze
+          FORM_FIELD_TAG_HOST = 'host'.freeze
+          FORM_FIELD_TAG_LANGUAGE = 'language'.freeze
+          FORM_FIELD_TAG_PID = 'process_id'.freeze
+          FORM_FIELD_TAG_PROFILER_VERSION = 'profiler_version'.freeze
+          FORM_FIELD_TAG_RUNTIME = 'runtime'.freeze
+          FORM_FIELD_TAG_RUNTIME_ENGINE = 'runtime_engine'.freeze
+          FORM_FIELD_TAG_RUNTIME_ID = 'runtime-id'.freeze
+          FORM_FIELD_TAG_RUNTIME_PLATFORM = 'runtime_platform'.freeze
+          FORM_FIELD_TAG_RUNTIME_VERSION = 'runtime_version'.freeze
+          FORM_FIELD_TAG_SERVICE = 'service'.freeze
+          FORM_FIELD_TAG_VERSION = 'version'.freeze
+          FORM_FIELD_TAGS = 'tags'.freeze
+          FORM_FIELD_INTAKE_VERSION = 'version'.freeze
+
+          HEADER_CONTENT_TYPE = 'Content-Type'.freeze
+          HEADER_CONTENT_TYPE_OCTET_STREAM = 'application/octet-stream'.freeze
+
+          FORM_FIELD_PPROF_DATA = 'data[rubyprofile.pprof]'.freeze
+          PPROF_DEFAULT_FILENAME = 'rubyprofile.pprof.gz'.freeze
+
+          FORM_FIELD_CODE_PROVENANCE_DATA = 'data[code-provenance.json]'.freeze
+          CODE_PROVENANCE_FILENAME = 'code-provenance.json.gz'.freeze
+        end
+      end
+    end
+  end
+end

data/lib/datadog/profiling/old_recorder.rb

@@ -0,0 +1,106 @@
+# typed: true
+
+require 'datadog/profiling/buffer'
+require 'datadog/profiling/encoding/profile'
+
+module Datadog
+  module Profiling
+    # Stores profiling events gathered by the `Stack` collector
+    class OldRecorder
+      attr_reader :max_size
+
+      def initialize(
+        event_classes,
+        max_size,
+        last_flush_time: Time.now.utc
+      )
+        @buffers = {}
+        @last_flush_time = last_flush_time
+        @max_size = max_size
+
+        # Add a buffer for each class
+        event_classes.each do |event_class|
+          @buffers[event_class] = Profiling::Buffer.new(max_size)
+        end
+
+        # Event classes can only be added ahead of time
+        @buffers.freeze
+      end
+
+      def [](event_class)
+        @buffers[event_class]
+      end
+
+      def push(events)
+        if events.is_a?(Array)
+          # Push multiple events
+          event_class = events.first.class
+          raise UnknownEventError, event_class unless @buffers.key?(event_class)
+
+          @buffers[event_class].concat(events)
+        else
+          # Push single event
+          event_class = events.class
+          raise UnknownEventError, event_class unless @buffers.key?(event_class)
+
+          @buffers[event_class].push(events)
+        end
+      end
+
+      def serialize
+        event_count = 0
+
+        event_groups, start, finish = update_time do
+          @buffers.collect do |event_class, buffer|
+            events = buffer.pop
+            next if events.empty?
+
+            event_count += events.length
+            EventGroup.new(event_class, events)
+          end.compact
+        end
+
+        return if event_count.zero? # We don't want to report empty profiles
+
+        encoded_pprof =
+          Datadog::Profiling::Encoding::Profile::Protobuf.encode(
+            event_count: event_count,
+            event_groups: event_groups,
+            start: start,
+            finish: finish,
+          )
+
+        [start, finish, encoded_pprof]
+      end
+
+      # NOTE: Remember that if the recorder is being accessed by multiple threads, this is an inherently racy operation.
+      def empty?
+        @buffers.values.all?(&:empty?)
+      end
+
+      # Error when event of an unknown type is used with the OldRecorder
+      class UnknownEventError < StandardError
+        attr_reader :event_class
+
+        def initialize(event_class)
+          @event_class = event_class
+        end
+
+        def message
+          @message ||= "Unknown event class '#{event_class}' for profiling recorder."
+        end
+      end
+
+      private
+
+      def update_time
+        start = @last_flush_time
+        result = yield
+        @last_flush_time = Time.now.utc
+
+        # Return event groups, start time, finish time
+        [result, start, @last_flush_time]
+      end
+    end
+  end
+end

data/lib/datadog/profiling/pprof/builder.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+# typed: true
+
+require 'datadog/profiling/flush'
+require 'datadog/profiling/pprof/message_set'
+require 'datadog/profiling/pprof/string_table'
+require 'datadog/core/utils/time'
+
+module Datadog
+  module Profiling
+    module Pprof
+      # Accumulates profile data and produces a Perftools::Profiles::Profile
+      class Builder
+        DEFAULT_ENCODING = 'UTF-8'
+        DESC_FRAME_OMITTED = 'frame omitted'
+        DESC_FRAMES_OMITTED = 'frames omitted'
+
+        attr_reader \
+          :functions,
+          :locations,
+          :mappings,
+          :sample_types,
+          :samples,
+          :string_table
+
+        def initialize
+          @functions = MessageSet.new(1)
+          @locations = initialize_locations_hash
+          @mappings = MessageSet.new(1)
+          @sample_types = MessageSet.new
+          @samples = []
+          @string_table = StringTable.new
+
+          # Cache this proc, since it's pretty expensive to keep recreating it
+          @build_function = method(:build_function).to_proc
+        end
+
+        # The locations hash maps unique BacktraceLocation instances to their corresponding pprof Location objects;
+        # there's a 1:1 correspondence, since BacktraceLocations were already deduped
+        def initialize_locations_hash
+          sequence = Core::Utils::Sequence.new(1)
+          Hash.new do |locations_hash, backtrace_location|
+            locations_hash[backtrace_location] = build_location(sequence.next, backtrace_location)
+          end
+        end
+
+        def encode_profile(profile)
+          Perftools::Profiles::Profile.encode(profile).force_encoding(DEFAULT_ENCODING)
+        end
+
+        def build_profile(start:, finish:)
+          start_ns = Core::Utils::Time.as_utc_epoch_ns(start)
+          finish_ns = Core::Utils::Time.as_utc_epoch_ns(finish)
+
+          Perftools::Profiles::Profile.new(
+            sample_type: @sample_types.messages,
+            sample: @samples,
+            mapping: @mappings.messages,
+            location: @locations.values,
+            function: @functions.messages,
+            string_table: @string_table.strings,
+            time_nanos: start_ns,
+            duration_nanos: finish_ns - start_ns,
+          )
+        end
+
+        def build_value_type(type, unit)
+          Perftools::Profiles::ValueType.new(
+            type: @string_table.fetch(type),
+            unit: @string_table.fetch(unit)
+          )
+        end
+
+        def build_locations(backtrace_locations, length)
+          locations = backtrace_locations.collect { |backtrace_location| @locations[backtrace_location] }
+
+          omitted = length - backtrace_locations.length
+
+          # Add placeholder stack frame if frames were truncated
+          if omitted > 0
+            desc = omitted == 1 ? DESC_FRAME_OMITTED : DESC_FRAMES_OMITTED
+            locations << @locations[Profiling::BacktraceLocation.new('', 0, "#{omitted} #{desc}")]
+          end
+
+          locations
+        end
+
+        def build_location(id, backtrace_location)
+          Perftools::Profiles::Location.new(
+            id: id,
+            line: [build_line(
+              @functions.fetch(
+                backtrace_location.path,
+                backtrace_location.base_label,
+                &@build_function
+              ).id,
+              backtrace_location.lineno
+            )]
+          )
+        end
+
+        def build_line(function_id, line_number)
+          Perftools::Profiles::Line.new(
+            function_id: function_id,
+            line: line_number
+          )
+        end
+
+        def build_function(id, filename, function_name)
+          Perftools::Profiles::Function.new(
+            id: id,
+            name: @string_table.fetch(function_name),
+            filename: @string_table.fetch(filename)
+          )
+        end
+
+        def build_mapping(id, filename)
+          Perftools::Profiles::Mapping.new(
+            id: id,
+            filename: @string_table.fetch(filename)
+          )
+        end
+      end
+    end
+  end
+end

data/lib/datadog/profiling/pprof/converter.rb

@@ -0,0 +1,104 @@
+# typed: true
+
+require 'datadog/profiling/ext'
+
+module Datadog
+  module Profiling
+    module Pprof
+      # Base class for converters that convert profiling events to pprof
+      class Converter
+        attr_reader \
+          :builder
+
+        # Override in child class to define sample types
+        # this converter uses when building samples.
+        def self.sample_value_types
+          raise NotImplementedError
+        end
+
+        def initialize(builder, sample_type_mappings)
+          @builder = builder
+          @sample_type_mappings = sample_type_mappings
+        end
+
+        def group_events(events)
+          # Event grouping in format:
+          # [key, EventGroup]
+          event_groups = {}
+
+          # Aggregate each event into a group
+          # with identical properties, but different values.
+          events.each do |event|
+            key = yield(event)
+            values = build_event_values(event)
+
+            unless key.nil?
+              if event_groups.key?(key)
+                # Update existing group from event
+                update_group(event_groups[key], event, values)
+              else
+                # Add new group
+                event_groups[key] = new_group(event, values)
+              end
+            end
+          end
+
+          event_groups
+        end
+
+        def add_events!(events)
+          raise NotImplementedError
+        end
+
+        def sample_value_index(type)
+          index = @sample_type_mappings[type]
+          raise UnknownSampleTypeMappingError, type unless index
+
+          index
+        end
+
+        def build_event_values(event)
+          # Build a value array that matches the length of the sample types
+          # Populate all values with "no value" by default
+          Array.new(@sample_type_mappings.length, Profiling::Ext::Pprof::SAMPLE_VALUE_NO_VALUE)
+        end
+
+        def debug_statistics
+          # Empty; can be used by subclasses to report a string containing debug statistics to be logged
+        end
+
+        # Represents a grouped event
+        # 'sample' is an example event object from the group.
+        # 'values' is the summation of the group's sample values
+        EventGroup = Struct.new(:sample, :values)
+
+        # Error when the mapping of a sample type to value index is unknown
+        class UnknownSampleTypeMappingError < StandardError
+          attr_reader :type
+
+          def initialize(type)
+            @type = type
+          end
+
+          def message
+            "Mapping for sample value type '#{type}' to index is unknown."
+          end
+        end
+
+        protected
+
+        def new_group(event, values)
+          EventGroup.new(event, values)
+        end
+
+        def update_group(event_group, event, values)
+          # Update values for group
+          group_values = event_group.values
+          group_values.each_with_index do |group_value, i|
+            group_values[i] = group_value + values[i]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/profiling/pprof/message_set.rb

@@ -0,0 +1,16 @@
+# typed: true
+
+require 'datadog/core/utils/object_set'
+
+module Datadog
+  module Profiling
+    module Pprof
+      # Acts as a unique dictionary of protobuf messages
+      class MessageSet < Core::Utils::ObjectSet
+        def messages
+          objects
+        end
+      end
+    end
+  end
+end

data/lib/datadog/profiling/pprof/payload.rb

@@ -0,0 +1,20 @@
+# typed: true
+
+module Datadog
+  module Profiling
+    module Pprof
+      # Pprof output data.
+      # Includes encoded data and list of types.
+      Payload = Struct.new(:data, :types) do
+        def initialize(data, types)
+          super
+          self.types = types || []
+        end
+
+        def to_s
+          data
+        end
+      end
+    end
+  end
+end