ddtrace 1.0.0 → 1.10.1

data/lib/datadog/profiling/http_transport.rb

@@ -0,0 +1,130 @@
+module Datadog
+  module Profiling
+    # Used to report profiling data to Datadog.
+    # Methods prefixed with _native_ are implemented in `http_transport.c`
+    class HttpTransport
+      def initialize(agent_settings:, site:, api_key:, upload_timeout_seconds:)
+        @upload_timeout_milliseconds = (upload_timeout_seconds * 1_000).to_i
+
+        validate_agent_settings(agent_settings)
+
+        @exporter_configuration =
+          if agentless?(site, api_key)
+            [:agentless, site, api_key]
+          else
+            [:agent, base_url_from(agent_settings)]
+          end
+
+        status, result = validate_exporter(@exporter_configuration)
+
+        raise(ArgumentError, "Failed to initialize transport: #{result}") if status == :error
+      end
+
+      def export(flush)
+        status, result = do_export(
+          exporter_configuration: @exporter_configuration,
+          upload_timeout_milliseconds: @upload_timeout_milliseconds,
+
+          # why "timespec"?
+          # libdatadog represents time using POSIX's struct timespec, see
+          # https://www.gnu.org/software/libc/manual/html_node/Time-Types.html
+          # aka it represents the seconds part separate from the nanoseconds part
+          start_timespec_seconds: flush.start.tv_sec,
+          start_timespec_nanoseconds: flush.start.tv_nsec,
+          finish_timespec_seconds: flush.finish.tv_sec,
+          finish_timespec_nanoseconds: flush.finish.tv_nsec,
+
+          pprof_file_name: flush.pprof_file_name,
+          pprof_data: flush.pprof_data,
+          code_provenance_file_name: flush.code_provenance_file_name,
+          code_provenance_data: flush.code_provenance_data,
+
+          tags_as_array: flush.tags_as_array,
+        )
+
+        if status == :ok
+          if (200..299).cover?(result)
+            Datadog.logger.debug('Successfully reported profiling data')
+            true
+          else
+            Datadog.logger.error("Failed to report profiling data: server returned unexpected HTTP #{result} status code")
+            false
+          end
+        else
+          Datadog.logger.error("Failed to report profiling data: #{result}")
+          false
+        end
+      end
+
+      # Used to log soft failures in `ddog_Vec_tag_push` (e.g. we still report the profile in these cases)
+      # Called from native code
+      def self.log_failure_to_process_tag(failure_details)
+        Datadog.logger.warn("Failed to add tag to profiling request: #{failure_details}")
+      end
+
+      private
+
+      def base_url_from(agent_settings)
+        case agent_settings.adapter
+        when Datadog::Transport::Ext::HTTP::ADAPTER
+          "#{agent_settings.ssl ? 'https' : 'http'}://#{agent_settings.hostname}:#{agent_settings.port}/"
+        when Datadog::Transport::Ext::UnixSocket::ADAPTER
+          "unix://#{agent_settings.uds_path}"
+        else
+          raise ArgumentError, "Unexpected adapter: #{agent_settings.adapter}"
+        end
+      end
+
+      def validate_agent_settings(agent_settings)
+        supported_adapters = [Datadog::Transport::Ext::HTTP::ADAPTER, Datadog::Transport::Ext::UnixSocket::ADAPTER]
+        unless supported_adapters.include?(agent_settings.adapter)
+          raise ArgumentError,
+            "Unsupported transport configuration for profiling: Adapter #{agent_settings.adapter} " \
+                        ' is not supported'
+        end
+
+        if agent_settings.deprecated_for_removal_transport_configuration_proc
+          Datadog.logger.warn(
+            'Ignoring custom c.tracing.transport_options setting as the profiler does not support it.'
+          )
+        end
+      end
+
+      def agentless?(site, api_key)
+        site && api_key && Core::Environment::VariableHelpers.env_to_bool(Profiling::Ext::ENV_AGENTLESS, false)
+      end
+
+      def validate_exporter(exporter_configuration)
+        self.class._native_validate_exporter(exporter_configuration)
+      end
+
+      def do_export(
+        exporter_configuration:,
+        upload_timeout_milliseconds:,
+        start_timespec_seconds:,
+        start_timespec_nanoseconds:,
+        finish_timespec_seconds:,
+        finish_timespec_nanoseconds:,
+        pprof_file_name:,
+        pprof_data:,
+        code_provenance_file_name:,
+        code_provenance_data:,
+        tags_as_array:
+      )
+        self.class._native_do_export(
+          exporter_configuration,
+          upload_timeout_milliseconds,
+          start_timespec_seconds,
+          start_timespec_nanoseconds,
+          finish_timespec_seconds,
+          finish_timespec_nanoseconds,
+          pprof_file_name,
+          pprof_data,
+          code_provenance_file_name,
+          code_provenance_data,
+          tags_as_array,
+        )
+      end
+    end
+  end
+end

data/lib/datadog/profiling/load_native_extension.rb

@@ -0,0 +1,20 @@
+# This file is used to load the profiling native extension. It works in two steps:
+#
+# 1. Load the ddtrace_profiling_loader extension. This extension will be used to load the actual extension, but in
+#    a special way that avoids exposing native-level code symbols. See `ddtrace_profiling_loader.c` for more details.
+#
+# 2. Use the Datadog::Profiling::Loader exposed by the ddtrace_profiling_loader extension to load the actual
+#    profiling native extension.
+#
+# All code on this file is on-purpose at the top-level; this makes it so this file is executed only once,
+# the first time it gets required, to avoid any issues with the native extension being initialized more than once.
+
+require "ddtrace_profiling_loader.#{RUBY_VERSION}_#{RUBY_PLATFORM}"
+
+extension_name = "ddtrace_profiling_native_extension.#{RUBY_VERSION}_#{RUBY_PLATFORM}"
+full_file_path = "#{__dir__}/../../#{extension_name}.#{RbConfig::CONFIG['DLEXT']}"
+init_function_name = "Init_#{extension_name.split('.').first}"
+
+status, result = Datadog::Profiling::Loader._native_load(full_file_path, init_function_name)
+
+raise "Failure to load #{extension_name} due to #{result}" if status == :error

data/lib/datadog/profiling/native_extension.rb

@@ -1,5 +1,3 @@
-# typed: true
-
 module Datadog
   module Profiling
     # This module contains classes and methods which are implemented using native code in the

data/lib/datadog/profiling/old_recorder.rb

@@ -0,0 +1,107 @@
+require_relative 'buffer'
+require_relative 'encoding/profile'
+
+module Datadog
+  module Profiling
+    # Stores profiling events gathered by the `Stack` collector
+    class OldRecorder
+      attr_reader :max_size
+
+      def initialize(
+        event_classes,
+        max_size,
+        last_flush_time: Time.now.utc
+      )
+        @buffers = {}
+        @last_flush_time = last_flush_time
+        @max_size = max_size
+
+        # Add a buffer for each class
+        event_classes.each do |event_class|
+          @buffers[event_class] = Profiling::Buffer.new(max_size)
+        end
+
+        # Event classes can only be added ahead of time
+        @buffers.freeze
+      end
+
+      def [](event_class)
+        @buffers[event_class]
+      end
+
+      def push(events)
+        if events.is_a?(Array)
+          # Push multiple events
+          event_class = events.first.class
+          raise UnknownEventError, event_class unless @buffers.key?(event_class)
+
+          @buffers[event_class].concat(events)
+        else
+          # Push single event
+          event_class = events.class
+          raise UnknownEventError, event_class unless @buffers.key?(event_class)
+
+          @buffers[event_class].push(events)
+        end
+      end
+
+      def serialize
+        event_count = 0
+
+        event_groups, start, finish = update_time do
+          @buffers.collect do |event_class, buffer|
+            events = buffer.pop
+            next if events.empty?
+
+            event_count += events.length
+            EventGroup.new(event_class, events)
+          end.compact
+        end
+
+        return if event_count.zero? # We don't want to report empty profiles
+
+        encoded_pprof =
+          Datadog::Profiling::Encoding::Profile::Protobuf.encode(
+            event_count: event_count,
+            event_groups: event_groups,
+            start: start,
+            finish: finish,
+          )
+
+        [start, finish, encoded_pprof]
+      end
+
+      def reset_after_fork
+        Datadog.logger.debug('Resetting OldRecorder in child process after fork')
+
+        # NOTE: A bit of a heavy-handed approach, but it doesn't happen often and this class will be removed soon anyway
+        serialize
+        nil
+      end
+
+      # Error when event of an unknown type is used with the OldRecorder
+      class UnknownEventError < StandardError
+        attr_reader :event_class
+
+        def initialize(event_class)
+          @event_class = event_class
+        end
+
+        def message
+          @message ||= "Unknown event class '#{event_class}' for profiling recorder."
+        end
+      end
+
+      private
+
+      def update_time
+        start = @last_flush_time
+        result = yield
+        @last_flush_time = Time.now.utc
+
+        # Return event groups, start time, finish time
+        [result, start, @last_flush_time]
+      end
+    end
+  end
+end

data/lib/datadog/profiling/pprof/builder.rb

@@ -1,11 +1,9 @@
 # frozen_string_literal: true
 
-# typed: true
-
-require 'datadog/profiling/flush'
-require 'datadog/profiling/pprof/message_set'
-require 'datadog/profiling/pprof/string_table'
-require 'datadog/core/utils/time'
+require_relative '../flush'
+require_relative 'message_set'
+require_relative 'string_table'
+require_relative '../../core/utils/time'
 
 module Datadog
   module Profiling

data/lib/datadog/profiling/pprof/converter.rb

@@ -1,6 +1,4 @@
-# typed: true
-
-require 'datadog/profiling/ext'
+require_relative '../ext'
 
 module Datadog
   module Profiling

data/lib/datadog/profiling/pprof/message_set.rb

@@ -1,6 +1,4 @@
-# typed: true
-
-require 'datadog/core/utils/object_set'
+require_relative '../../core/utils/object_set'
 
 module Datadog
   module Profiling

data/lib/datadog/profiling/pprof/payload.rb

@@ -1,5 +1,3 @@
-# typed: true
-
 module Datadog
   module Profiling
     module Pprof

data/lib/datadog/profiling/pprof/pprof_pb.rb

@@ -1,5 +1,3 @@
-# typed: false
-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: lib/datadog/profiling/pprof/pprof.proto
 

data/lib/datadog/profiling/pprof/stack_sample.rb

@@ -1,9 +1,7 @@
-# typed: true
-
-require 'datadog/profiling/ext'
-require 'datadog/profiling/events/stack'
-require 'datadog/profiling/pprof/builder'
-require 'datadog/profiling/pprof/converter'
+require_relative '../ext'
+require_relative '../events/stack'
+require_relative 'builder'
+require_relative 'converter'
 
 module Datadog
   module Profiling

data/lib/datadog/profiling/pprof/string_table.rb

@@ -1,6 +1,4 @@
-# typed: strict
-
-require 'datadog/core/utils/string_table'
+require_relative '../../core/utils/string_table'
 
 module Datadog
   module Profiling

data/lib/datadog/profiling/pprof/template.rb

@@ -1,11 +1,9 @@
-# typed: false
+require_relative 'payload'
+require_relative 'message_set'
+require_relative 'builder'
 
-require 'datadog/profiling/pprof/payload'
-require 'datadog/profiling/pprof/message_set'
-require 'datadog/profiling/pprof/builder'
-
-require 'datadog/profiling/events/stack'
-require 'datadog/profiling/pprof/stack_sample'
+require_relative '../events/stack'
+require_relative 'stack_sample'
 
 module Datadog
   module Profiling

data/lib/datadog/profiling/preload.rb

@@ -1,5 +1,3 @@
-# typed: strict
-
-require 'ddtrace'
+require_relative '../../ddtrace'
 
 Datadog::Profiling.start_if_enabled

data/lib/datadog/profiling/profiler.rb

@@ -1,9 +1,9 @@
-# typed: true
-
 module Datadog
   module Profiling
     # Profiling entry point, which coordinates collectors and a scheduler
     class Profiler
+      include Datadog::Core::Utils::Forking
+
       attr_reader \
         :collectors,
         :scheduler
@@ -14,6 +14,11 @@ module Datadog
       end
 
       def start
+        after_fork! do
+          collectors.each(&:reset_after_fork)
+          scheduler.reset_after_fork
+        end
+
         collectors.each(&:start)
         scheduler.start
       end

data/lib/datadog/profiling/scheduler.rb

@@ -1,42 +1,39 @@
-# typed: true
+require_relative '../core/utils/time'
 
-require 'datadog/core/utils/time'
-
-require 'datadog/core/worker'
-require 'datadog/core/workers/polling'
+require_relative '../core/worker'
+require_relative '../core/workers/polling'
 
 module Datadog
   module Profiling
-    # Periodically (every DEFAULT_INTERVAL_SECONDS) takes data from the `Recorder` and pushes them to all configured
-    # `Exporter`s. Runs on its own background thread.
+    # Periodically (every DEFAULT_INTERVAL_SECONDS) takes a profile from the `Exporter` and reports it using the
+    # configured transport. Runs on its own background thread.
     class Scheduler < Core::Worker
       include Core::Workers::Polling
 
       DEFAULT_INTERVAL_SECONDS = 60
       MINIMUM_INTERVAL_SECONDS = 0
 
-      # Profiles with duration less than this will not be reported
-      PROFILE_DURATION_THRESHOLD_SECONDS = 1
-
       # We sleep for at most this duration seconds before reporting data to avoid multi-process applications all
       # reporting profiles at the exact same time
       DEFAULT_FLUSH_JITTER_MAXIMUM_SECONDS = 3
 
-      private_constant :DEFAULT_INTERVAL_SECONDS, :MINIMUM_INTERVAL_SECONDS, :PROFILE_DURATION_THRESHOLD_SECONDS
+      private
 
       attr_reader \
-        :exporters,
-        :recorder
+        :exporter,
+        :transport
+
+      public
 
       def initialize(
-        recorder,
-        exporters,
+        exporter:,
+        transport:,
         fork_policy: Core::Workers::Async::Thread::FORK_POLICY_RESTART, # Restart in forks by default
         interval: DEFAULT_INTERVAL_SECONDS,
         enabled: true
       )
-        @recorder = recorder
-        @exporters = [exporters].flatten
+        @exporter = exporter
+        @transport = transport
 
         # Workers::Async::Thread settings
         self.fork_policy = fork_policy
@@ -67,24 +64,21 @@ module Datadog
         end
       end
 
-      def after_fork
-        # Clear recorder's buffers by flushing events.
-        # Objects from parent process will copy-on-write,
-        # and we don't want to send events for the wrong process.
-        recorder.flush
-      end
-
       # Configure Workers::IntervalLoop to not report immediately when scheduler starts
       #
       # When a scheduler gets created (or reset), we don't want it to immediately try to flush; we want it to wait for
       # the loop wait time first. This avoids an issue where the scheduler reported a mostly-empty profile if the
-      # application just started but this thread took a bit longer so there's already samples in the recorder.
+      # application just started but this thread took a bit longer so there's already profiling data in the exporter.
       def loop_wait_before_first_iteration?
         true
       end
 
       def work_pending?
-        !recorder.empty?
+        exporter.can_flush?
+      end
+
+      def reset_after_fork
+        exporter.reset_after_fork
       end
 
       private
@@ -100,21 +94,15 @@ module Datadog
       end
 
       def flush_events
-        # Get events from recorder
-        flush = recorder.flush
-
-        if duration_below_threshold?(flush)
-          Datadog.logger.debug do
-            "Skipped exporting profiling events as profile duration is below minimum (#{flush.event_count} events skipped)"
-          end
+        # Collect data to be exported
+        flush = exporter.flush
 
-          return flush
-        end
+        return false unless flush
 
         # Sleep for a bit to cause misalignment between profilers in multi-process applications
         #
         # When not being run in a loop, it means the scheduler has not been started or was stopped, and thus
-        # a) it's being shutting down (and is trying to report the last profile)
+        # a) it's being shut down (and is trying to report the last profile)
         # b) it's being run as a one-shot, usually in a test
         # ...so in those cases we don't sleep
         #
@@ -127,24 +115,13 @@ module Datadog
           sleep(jitter_seconds)
         end
 
-        # Send events to each exporter
-        if flush.event_count > 0
-          exporters.each do |exporter|
-            begin
-              exporter.export(flush)
-            rescue StandardError => e
-              Datadog.logger.error(
-                "Unable to export #{flush.event_count} profiling events. Cause: #{e} Location: #{Array(e.backtrace).first}"
-              )
-            end
-          end
+        begin
+          transport.export(flush)
+        rescue StandardError => e
+          Datadog.logger.error("Unable to report profile. Cause: #{e} Location: #{Array(e.backtrace).first}")
         end
 
-        flush
-      end
-
-      def duration_below_threshold?(flush)
-        (flush.finish - flush.start) < PROFILE_DURATION_THRESHOLD_SECONDS
+        true
       end
     end
   end

data/lib/datadog/profiling/stack_recorder.rb

@@ -0,0 +1,74 @@
+module Datadog
+  module Profiling
+    # Stores stack samples in a native libdatadog data structure and expose Ruby-level serialization APIs
+    # Note that `record_sample` is only accessible from native code.
+    # Methods prefixed with _native_ are implemented in `stack_recorder.c`
+    class StackRecorder
+      def initialize(cpu_time_enabled:, alloc_samples_enabled:)
+        # This mutex works in addition to the fancy C-level mutexes we have in the native side (see the docs there).
+        # It prevents multiple Ruby threads calling serialize at the same time -- something like
+        # `10.times { Thread.new { stack_recorder.serialize } }`.
+        # This isn't something we expect to happen normally, but because it would break the assumptions of the
+        # C-level mutexes (that there is a single serializer thread), we add it here as an extra safeguard against it
+        # accidentally happening.
+        @no_concurrent_synchronize_mutex = Mutex.new
+
+        self.class._native_initialize(self, cpu_time_enabled, alloc_samples_enabled)
+      end
+
+      def serialize
+        status, result = @no_concurrent_synchronize_mutex.synchronize { self.class._native_serialize(self) }
+
+        if status == :ok
+          start, finish, encoded_pprof = result
+
+          Datadog.logger.debug { "Encoded profile covering #{start.iso8601} to #{finish.iso8601}" }
+
+          [start, finish, encoded_pprof]
+        else
+          error_message = result
+
+          Datadog.logger.error("Failed to serialize profiling data: #{error_message}")
+
+          nil
+        end
+      end
+
+      def serialize!
+        status, result = @no_concurrent_synchronize_mutex.synchronize { self.class._native_serialize(self) }
+
+        if status == :ok
+          _start, _finish, encoded_pprof = result
+
+          encoded_pprof
+        else
+          error_message = result
+
+          raise("Failed to serialize profiling data: #{error_message}")
+        end
+      end
+
+      def clear
+        status, result = @no_concurrent_synchronize_mutex.synchronize { self.class._native_clear(self) }
+
+        if status == :ok
+          finish_timestamp = result
+
+          Datadog.logger.debug { "Cleared profile at #{finish_timestamp}" }
+
+          finish_timestamp
+        else
+          error_message = result
+
+          Datadog.logger.error("Failed to clear profiling data: #{error_message}")
+
+          nil
+        end
+      end
+
+      def reset_after_fork
+        self.class._native_reset_after_fork(self)
+      end
+    end
+  end
+end

data/lib/datadog/profiling/tag_builder.rb

@@ -0,0 +1,51 @@
+require_relative '../core/utils'
+
+module Datadog
+  module Profiling
+    # Builds a hash of default plus user tags to be included in a profile
+    module TagBuilder
+      include Datadog::Profiling::Ext::Transport::HTTP # Tag name constants
+
+      def self.call(
+        settings:,
+        # Unified service tagging
+        env: settings.env,
+        service: settings.service,
+        version: settings.version,
+        # Other metadata
+        host: Core::Environment::Socket.hostname,
+        language: Core::Environment::Identity.lang,
+        pid: Process.pid.to_s,
+        profiler_version: Core::Environment::Identity.tracer_version,
+        runtime_engine: Core::Environment::Identity.lang_engine,
+        runtime_id: Core::Environment::Identity.id,
+        runtime_platform: Core::Environment::Identity.lang_platform,
+        runtime_version: Core::Environment::Identity.lang_version,
+        # User-provided tags
+        user_tags: settings.tags
+      )
+        tags = {
+          # When changing or adding these, make sure they are kept in sync with
+          # https://docs.google.com/spreadsheets/d/1LOGMf4c4Avbtn36uZ2SWvhIGKRPLM1BoWkUP4JYj7hA/ (Datadog internal link)
+          FORM_FIELD_TAG_HOST => host,
+          FORM_FIELD_TAG_LANGUAGE => language,
+          FORM_FIELD_TAG_PID => pid,
+          FORM_FIELD_TAG_PROFILER_VERSION => profiler_version,
+          FORM_FIELD_TAG_RUNTIME => language, # This is known to be repeated from language, above
+          FORM_FIELD_TAG_RUNTIME_ENGINE => runtime_engine,
+          FORM_FIELD_TAG_RUNTIME_ID => runtime_id,
+          FORM_FIELD_TAG_RUNTIME_PLATFORM => runtime_platform,
+          FORM_FIELD_TAG_RUNTIME_VERSION => runtime_version,
+        }
+        tags[FORM_FIELD_TAG_ENV] = env if env
+        tags[FORM_FIELD_TAG_SERVICE] = service if service
+        tags[FORM_FIELD_TAG_VERSION] = version if version
+
+        # Make sure everything is an utf-8 string, to avoid encoding issues in native code/libddprof/further downstream
+        user_tags.merge(tags).map do |key, value|
+          [Datadog::Core::Utils.utf8_encode(key), Datadog::Core::Utils.utf8_encode(value)]
+        end.to_h
+      end
+    end
+  end
+end

data/lib/datadog/profiling/tasks/exec.rb

@@ -1,5 +1,3 @@
-# typed: true
-
 module Datadog
   module Profiling
     module Tasks
@@ -38,10 +36,10 @@ module Datadog
         def exec_with_error_handling(args)
           Kernel.exec(*args)
         rescue Errno::ENOENT => e
-          Kernel.warn "ddtracerb exec failed: #{e.message} (command was '#{args.join(' ')}')"
+          Kernel.warn "ddtracerb exec failed: #{e.class.name} #{e.message} (command was '#{args.join(' ')}')"
           Kernel.exit 127
         rescue Errno::EACCES, Errno::ENOEXEC => e
-          Kernel.warn "ddtracerb exec failed: #{e.message} (command was '#{args.join(' ')}')"
+          Kernel.warn "ddtracerb exec failed: #{e.class.name} #{e.message} (command was '#{args.join(' ')}')"
           Kernel.exit 126
         end
       end