ddtrace 1.1.0 → 1.2.0

data/lib/datadog/profiling/flush.rb

@@ -5,65 +5,37 @@ require 'datadog/core/environment/socket'
 
 module Datadog
   module Profiling
+    # Represents a collection of events of a specific type being flushed.
+    EventGroup = Struct.new(:event_class, :events)
+
     # Entity class used to represent metadata for a given profile
-    OldFlush = Struct.new(
-      :start,
-      :finish,
-      :event_groups,
-      :event_count,
-      :code_provenance,
-      :runtime_id,
-      :service,
-      :env,
-      :version,
-      :host,
-      :language,
-      :runtime_engine,
-      :runtime_platform,
-      :runtime_version,
-      :profiler_version,
-      :tags
-    ) do
+    class Flush
+      attr_reader \
+        :start,
+        :finish,
+        :pprof_file_name,
+        :pprof_data, # gzipped pprof bytes
+        :code_provenance_file_name,
+        :code_provenance_data, # gzipped json bytes
+        :tags_as_array
+
       def initialize(
         start:,
         finish:,
-        event_groups:,
-        event_count:,
-        code_provenance:,
-        runtime_id: Core::Environment::Identity.id,
-        service: Datadog.configuration.service,
-        env: Datadog.configuration.env,
-        version: Datadog.configuration.version,
-        host: Core::Environment::Socket.hostname,
-        language: Core::Environment::Identity.lang,
-        runtime_engine: Core::Environment::Identity.lang_engine,
-        runtime_platform: Core::Environment::Identity.lang_platform,
-        runtime_version: Core::Environment::Identity.lang_version,
-        profiler_version: Core::Environment::Identity.tracer_version,
-        tags: Datadog.configuration.tags
+        pprof_file_name:,
+        pprof_data:,
+        code_provenance_file_name:,
+        code_provenance_data:,
+        tags_as_array:
       )
-        super(
-          start,
-          finish,
-          event_groups,
-          event_count,
-          code_provenance,
-          runtime_id,
-          service,
-          env,
-          version,
-          host,
-          language,
-          runtime_engine,
-          runtime_platform,
-          runtime_version,
-          profiler_version,
-          tags,
-        )
+        @start = start
+        @finish = finish
+        @pprof_file_name = pprof_file_name
+        @pprof_data = pprof_data
+        @code_provenance_file_name = code_provenance_file_name
+        @code_provenance_data = code_provenance_data
+        @tags_as_array = tags_as_array
       end
     end
-
-    # Represents a collection of events of a specific type being flushed.
-    EventGroup = Struct.new(:event_class, :events).freeze
   end
 end

data/lib/datadog/profiling/http_transport.rb

@@ -0,0 +1,131 @@
+# typed: false
+
+module Datadog
+  module Profiling
+    # Used to report profiling data to Datadog.
+    # Methods prefixed with _native_ are implemented in `http_transport.c`
+    class HttpTransport
+      def initialize(agent_settings:, site:, api_key:, upload_timeout_seconds:)
+        @upload_timeout_milliseconds = (upload_timeout_seconds * 1_000).to_i
+
+        validate_agent_settings(agent_settings)
+
+        @exporter_configuration =
+          if agentless?(site, api_key)
+            [:agentless, site, api_key]
+          else
+            [:agent, base_url_from(agent_settings)]
+          end
+
+        status, result = validate_exporter(@exporter_configuration)
+
+        raise(ArgumentError, "Failed to initialize transport: #{result}") if status == :error
+      end
+
+      def export(flush)
+        status, result = do_export(
+          exporter_configuration: @exporter_configuration,
+          upload_timeout_milliseconds: @upload_timeout_milliseconds,
+
+          # why "timespec"?
+          # libddprof represents time using POSIX's struct timespec, see
+          # https://www.gnu.org/software/libc/manual/html_node/Time-Types.html
+          # aka it represents the seconds part separate from the nanoseconds part
+          start_timespec_seconds: flush.start.tv_sec,
+          start_timespec_nanoseconds: flush.start.tv_nsec,
+          finish_timespec_seconds: flush.finish.tv_sec,
+          finish_timespec_nanoseconds: flush.finish.tv_nsec,
+
+          pprof_file_name: flush.pprof_file_name,
+          pprof_data: flush.pprof_data,
+          code_provenance_file_name: flush.code_provenance_file_name,
+          code_provenance_data: flush.code_provenance_data,
+
+          tags_as_array: flush.tags_as_array,
+        )
+
+        if status == :ok
+          if (200..299).cover?(result)
+            Datadog.logger.debug('Successfully reported profiling data')
+            true
+          else
+            Datadog.logger.error("Failed to report profiling data: server returned unexpected HTTP #{result} status code")
+            false
+          end
+        else
+          Datadog.logger.error("Failed to report profiling data: #{result}")
+          false
+        end
+      end
+
+      # Used to log soft failures in `ddprof_ffi_Vec_tag_push` (e.g. we still report the profile in these cases)
+      # Called from native code
+      def self.log_failure_to_process_tag(failure_details)
+        Datadog.logger.warn("Failed to add tag to profiling request: #{failure_details}")
+      end
+
+      private
+
+      def base_url_from(agent_settings)
+        case agent_settings.adapter
+        when Datadog::Transport::Ext::HTTP::ADAPTER
+          "#{agent_settings.ssl ? 'https' : 'http'}://#{agent_settings.hostname}:#{agent_settings.port}/"
+        when Datadog::Transport::Ext::UnixSocket::ADAPTER
+          "unix://#{agent_settings.uds_path}"
+        else
+          raise ArgumentError, "Unexpected adapter: #{agent_settings.adapter}"
+        end
+      end
+
+      def validate_agent_settings(agent_settings)
+        supported_adapters = [Datadog::Transport::Ext::HTTP::ADAPTER, Datadog::Transport::Ext::UnixSocket::ADAPTER]
+        unless supported_adapters.include?(agent_settings.adapter)
+          raise ArgumentError, "Unsupported transport configuration for profiling: Adapter #{agent_settings.adapter} " \
+            ' is not supported'
+        end
+
+        if agent_settings.deprecated_for_removal_transport_configuration_proc
+          Datadog.logger.warn(
+            'Ignoring custom c.tracing.transport_options setting as the profiler does not support it.'
+          )
+        end
+      end
+
+      def agentless?(site, api_key)
+        site && api_key && Core::Environment::VariableHelpers.env_to_bool(Profiling::Ext::ENV_AGENTLESS, false)
+      end
+
+      def validate_exporter(exporter_configuration)
+        self.class._native_validate_exporter(exporter_configuration)
+      end
+
+      def do_export(
+        exporter_configuration:,
+        upload_timeout_milliseconds:,
+        start_timespec_seconds:,
+        start_timespec_nanoseconds:,
+        finish_timespec_seconds:,
+        finish_timespec_nanoseconds:,
+        pprof_file_name:,
+        pprof_data:,
+        code_provenance_file_name:,
+        code_provenance_data:,
+        tags_as_array:
+      )
+        self.class._native_do_export(
+          exporter_configuration,
+          upload_timeout_milliseconds,
+          start_timespec_seconds,
+          start_timespec_nanoseconds,
+          finish_timespec_seconds,
+          finish_timespec_nanoseconds,
+          pprof_file_name,
+          pprof_data,
+          code_provenance_file_name,
+          code_provenance_data,
+          tags_as_array,
+        )
+      end
+    end
+  end
+end

data/lib/datadog/profiling/old_ext.rb

@@ -0,0 +1,42 @@
+# typed: true
+
+module Datadog
+  module Profiling
+    # NOTE: This OldExt file is temporary and expected to be removed once the migration to the new `HttpTransport` class
+    # is complete
+    module OldExt
+      module Transport
+        module HTTP
+          URI_TEMPLATE_DD_API = 'https://intake.profile.%s/'.freeze
+
+          FORM_FIELD_RECORDING_START = 'start'.freeze
+          FORM_FIELD_RECORDING_END = 'end'.freeze
+          FORM_FIELD_FAMILY = 'family'.freeze
+          FORM_FIELD_TAG_ENV = 'env'.freeze
+          FORM_FIELD_TAG_HOST = 'host'.freeze
+          FORM_FIELD_TAG_LANGUAGE = 'language'.freeze
+          FORM_FIELD_TAG_PID = 'process_id'.freeze
+          FORM_FIELD_TAG_PROFILER_VERSION = 'profiler_version'.freeze
+          FORM_FIELD_TAG_RUNTIME = 'runtime'.freeze
+          FORM_FIELD_TAG_RUNTIME_ENGINE = 'runtime_engine'.freeze
+          FORM_FIELD_TAG_RUNTIME_ID = 'runtime-id'.freeze
+          FORM_FIELD_TAG_RUNTIME_PLATFORM = 'runtime_platform'.freeze
+          FORM_FIELD_TAG_RUNTIME_VERSION = 'runtime_version'.freeze
+          FORM_FIELD_TAG_SERVICE = 'service'.freeze
+          FORM_FIELD_TAG_VERSION = 'version'.freeze
+          FORM_FIELD_TAGS = 'tags'.freeze
+          FORM_FIELD_INTAKE_VERSION = 'version'.freeze
+
+          HEADER_CONTENT_TYPE = 'Content-Type'.freeze
+          HEADER_CONTENT_TYPE_OCTET_STREAM = 'application/octet-stream'.freeze
+
+          FORM_FIELD_PPROF_DATA = 'data[rubyprofile.pprof]'.freeze
+          PPROF_DEFAULT_FILENAME = 'rubyprofile.pprof.gz'.freeze
+
+          FORM_FIELD_CODE_PROVENANCE_DATA = 'data[code-provenance.json]'.freeze
+          CODE_PROVENANCE_FILENAME = 'code-provenance.json.gz'.freeze
+        end
+      end
+    end
+  end
+end

data/lib/datadog/profiling/{recorder.rb → old_recorder.rb}

@@ -1,35 +1,22 @@
 # typed: true
 
 require 'datadog/profiling/buffer'
-require 'datadog/profiling/flush'
+require 'datadog/profiling/encoding/profile'
 
 module Datadog
   module Profiling
-    # Stores profiling events gathered by `Collector`s
-    class Recorder
+    # Stores profiling events gathered by the `Stack` collector
+    class OldRecorder
       attr_reader :max_size
 
-      # TODO: Why does the Recorder directly reference the `code_provenance_collector`?
-      #
-      # For starters, this is weird/a problem because the relationship is supposed to go in the other direction:
-      # collectors are supposed to record their results in the `Recorder`, rather than the `Recorder` having to know
-      # about individual collectors.
-      #
-      # But the `code_provenance_collector` is different from other existing and potential collectors because it is not
-      # asynchronous. It does not gather data over time and record it as it goes. Instead, you call it once per profile,
-      # synchronously, and just use what it spits out.
-      #
-      # The current design of the `Recorder` is quite tied to the asynchronous model. Modifying our current design
-      # to support synchronous collectors is non-trivial, and I decided not to go through with it because we're
-      # soon going to replace the `Recorder` and many other existing classes with a
-      # [libddprof](https://github.com/datadog/libddprof)-based implementation, and thus I don't think massive refactors
-      # are worth it before moving to libddprof.
-
-      def initialize(event_classes, max_size, code_provenance_collector:, last_flush_time: Time.now.utc)
+      def initialize(
+        event_classes,
+        max_size,
+        last_flush_time: Time.now.utc
+      )
         @buffers = {}
         @last_flush_time = last_flush_time
         @max_size = max_size
-        @code_provenance_collector = code_provenance_collector
 
         # Add a buffer for each class
         event_classes.each do |event_class|
@@ -60,7 +47,7 @@ module Datadog
         end
       end
 
-      def flush
+      def serialize
         event_count = 0
 
         event_groups, start, finish = update_time do
@@ -73,15 +60,17 @@ module Datadog
           end.compact
         end
 
-        code_provenance = @code_provenance_collector.refresh.generate_json if @code_provenance_collector
+        return if event_count.zero? # We don't want to report empty profiles
 
-        OldFlush.new(
-          start: start,
-          finish: finish,
-          event_groups: event_groups,
-          event_count: event_count,
-          code_provenance: code_provenance,
-        )
+        encoded_pprof =
+          Datadog::Profiling::Encoding::Profile::Protobuf.encode(
+            event_count: event_count,
+            event_groups: event_groups,
+            start: start,
+            finish: finish,
+          )
+
+        [start, finish, encoded_pprof]
       end
 
       # NOTE: Remember that if the recorder is being accessed by multiple threads, this is an inherently racy operation.
@@ -89,7 +78,7 @@ module Datadog
         @buffers.values.all?(&:empty?)
       end
 
-      # Error when event of an unknown type is used with the Recorder
+      # Error when event of an unknown type is used with the OldRecorder
       class UnknownEventError < StandardError
         attr_reader :event_class
 

data/lib/datadog/profiling/scheduler.rb

@@ -7,36 +7,35 @@ require 'datadog/core/workers/polling'
 
 module Datadog
   module Profiling
-    # Periodically (every DEFAULT_INTERVAL_SECONDS) takes data from the `Recorder` and pushes them to all configured
-    # `Exporter`s. Runs on its own background thread.
+    # Periodically (every DEFAULT_INTERVAL_SECONDS) takes a profile from the `Exporter` and reports it using the
+    # configured transport. Runs on its own background thread.
     class Scheduler < Core::Worker
       include Core::Workers::Polling
 
       DEFAULT_INTERVAL_SECONDS = 60
       MINIMUM_INTERVAL_SECONDS = 0
 
-      # Profiles with duration less than this will not be reported
-      PROFILE_DURATION_THRESHOLD_SECONDS = 1
-
       # We sleep for at most this duration seconds before reporting data to avoid multi-process applications all
       # reporting profiles at the exact same time
       DEFAULT_FLUSH_JITTER_MAXIMUM_SECONDS = 3
 
-      private_constant :DEFAULT_INTERVAL_SECONDS, :MINIMUM_INTERVAL_SECONDS, :PROFILE_DURATION_THRESHOLD_SECONDS
+      private
 
       attr_reader \
-        :exporters,
-        :recorder
+        :exporter,
+        :transport
+
+      public
 
       def initialize(
-        recorder,
-        exporters,
+        exporter:,
+        transport:,
         fork_policy: Core::Workers::Async::Thread::FORK_POLICY_RESTART, # Restart in forks by default
         interval: DEFAULT_INTERVAL_SECONDS,
         enabled: true
       )
-        @recorder = recorder
-        @exporters = [exporters].flatten
+        @exporter = exporter
+        @transport = transport
 
         # Workers::Async::Thread settings
         self.fork_policy = fork_policy
@@ -68,23 +67,22 @@ module Datadog
       end
 
       def after_fork
-        # Clear recorder's buffers by flushing events.
-        # Objects from parent process will copy-on-write,
-        # and we don't want to send events for the wrong process.
-        recorder.flush
+        # Clear any existing profiling state.
+        # We don't want the child process to report profiling data from its parent.
+        exporter.flush
       end
 
       # Configure Workers::IntervalLoop to not report immediately when scheduler starts
       #
       # When a scheduler gets created (or reset), we don't want it to immediately try to flush; we want it to wait for
       # the loop wait time first. This avoids an issue where the scheduler reported a mostly-empty profile if the
-      # application just started but this thread took a bit longer so there's already samples in the recorder.
+      # application just started but this thread took a bit longer so there's already profiling data in the exporter.
       def loop_wait_before_first_iteration?
         true
       end
 
       def work_pending?
-        !recorder.empty?
+        !exporter.empty?
       end
 
       private
@@ -100,16 +98,10 @@ module Datadog
       end
 
       def flush_events
-        # Get events from recorder
-        flush = recorder.flush
+        # Collect data to be exported
+        flush = exporter.flush
 
-        if duration_below_threshold?(flush)
-          Datadog.logger.debug do
-            "Skipped exporting profiling events as profile duration is below minimum (#{flush.event_count} events skipped)"
-          end
-
-          return flush
-        end
+        return false unless flush
 
         # Sleep for a bit to cause misalignment between profilers in multi-process applications
         #
@@ -127,24 +119,13 @@ module Datadog
           sleep(jitter_seconds)
         end
 
-        # Send events to each exporter
-        if flush.event_count > 0
-          exporters.each do |exporter|
-            begin
-              exporter.export(flush)
-            rescue StandardError => e
-              Datadog.logger.error(
-                "Unable to export #{flush.event_count} profiling events. Cause: #{e} Location: #{Array(e.backtrace).first}"
-              )
-            end
-          end
+        begin
+          transport.export(flush)
+        rescue StandardError => e
+          Datadog.logger.error("Unable to report profile. Cause: #{e} Location: #{Array(e.backtrace).first}")
         end
 
-        flush
-      end
-
-      def duration_below_threshold?(flush)
-        (flush.finish - flush.start) < PROFILE_DURATION_THRESHOLD_SECONDS
+        true
       end
     end
   end

data/lib/datadog/profiling/transport/http/api/endpoint.rb

@@ -2,7 +2,7 @@
 
 require 'datadog/core/utils/compression'
 require 'datadog/core/vendor/multipart-post/multipart/post/composite_read_io'
-require 'datadog/profiling/ext'
+require 'datadog/profiling/old_ext'
 require 'datadog/profiling/transport/http/response'
 require 'ddtrace/transport/http/api/endpoint'
 
@@ -13,7 +13,7 @@ module Datadog
         module API
           # Datadog API endpoint for profiling
           class Endpoint < Datadog::Transport::HTTP::API::Endpoint
-            include Profiling::Ext::Transport::HTTP
+            include Profiling::OldExt::Transport::HTTP
 
             # These tags are read from the flush object (see below) directly and so we ignore any extra copies that
             # may come in the tags hash to avoid duplicates.
@@ -23,7 +23,7 @@ module Datadog
             attr_reader \
               :encoder
 
-            def initialize(path, encoder)
+            def initialize(path, encoder = nil)
               super(:post, path)
               @encoder = encoder
             end
@@ -40,48 +40,26 @@ module Datadog
             end
 
             def build_form(env)
-              flush = env.request.parcel.data
+              flush = env.request
               pprof_file = build_pprof(flush)
 
               form = {
                 FORM_FIELD_INTAKE_VERSION => '3', # Aka 1.3 intake format
                 FORM_FIELD_RECORDING_START => flush.start.utc.iso8601,
                 FORM_FIELD_RECORDING_END => flush.finish.utc.iso8601,
-                FORM_FIELD_TAGS => [
-                  "#{FORM_FIELD_TAG_RUNTIME}:#{flush.language}",
-                  "#{FORM_FIELD_TAG_RUNTIME_ID}:#{flush.runtime_id}",
-                  "#{FORM_FIELD_TAG_RUNTIME_ENGINE}:#{flush.runtime_engine}",
-                  "#{FORM_FIELD_TAG_RUNTIME_PLATFORM}:#{flush.runtime_platform}",
-                  "#{FORM_FIELD_TAG_RUNTIME_VERSION}:#{flush.runtime_version}",
-                  "#{FORM_FIELD_TAG_PID}:#{Process.pid}",
-                  "#{FORM_FIELD_TAG_PROFILER_VERSION}:#{flush.profiler_version}",
-                  # NOTE: Redundant w/ 'runtime'; may want to remove this later.
-                  "#{FORM_FIELD_TAG_LANGUAGE}:#{flush.language}",
-                  "#{FORM_FIELD_TAG_HOST}:#{flush.host}",
-                  *flush
-                    .tags
-                    .reject { |tag_key| TAGS_TO_IGNORE_IN_TAGS_HASH.include?(tag_key) }
-                    .map { |tag_key, tag_value| "#{tag_key}:#{tag_value}" }
-                ],
+                FORM_FIELD_TAGS => flush.tags_as_array.map { |key, value| "#{key}:#{value}" },
                 FORM_FIELD_PPROF_DATA => pprof_file,
-                FORM_FIELD_FAMILY => flush.language,
+                FORM_FIELD_FAMILY => 'ruby',
               }
 
-              # Optional fields
-              form[FORM_FIELD_TAGS] << "#{FORM_FIELD_TAG_SERVICE}:#{flush.service}" unless flush.service.nil?
-              form[FORM_FIELD_TAGS] << "#{FORM_FIELD_TAG_ENV}:#{flush.env}" unless flush.env.nil?
-              form[FORM_FIELD_TAGS] << "#{FORM_FIELD_TAG_VERSION}:#{flush.version}" unless flush.version.nil?
-
               # May not be available/enabled
-              form[FORM_FIELD_CODE_PROVENANCE_DATA] = build_code_provenance(flush) if flush.code_provenance
+              form[FORM_FIELD_CODE_PROVENANCE_DATA] = build_code_provenance(flush) if flush.code_provenance_data
 
               form
             end
 
             def build_pprof(flush)
-              pprof = encoder.encode(flush)
-
-              gzipped_pprof_data = Core::Utils::Compression.gzip(pprof.data)
+              gzipped_pprof_data = flush.pprof_data
 
               Core::Vendor::Multipart::Post::UploadIO.new(
                 StringIO.new(gzipped_pprof_data),
@@ -91,7 +69,7 @@ module Datadog
             end
 
             def build_code_provenance(flush)
-              gzipped_code_provenance = Core::Utils::Compression.gzip(flush.code_provenance)
+              gzipped_code_provenance = flush.code_provenance_data
 
               Core::Vendor::Multipart::Post::UploadIO.new(
                 StringIO.new(gzipped_code_provenance),

data/lib/datadog/profiling/transport/http/client.rb

@@ -1,7 +1,7 @@
 # typed: true
 
 require 'ddtrace/transport/http/client'
-require 'datadog/profiling/transport/client'
+require 'ddtrace/transport/request'
 
 module Datadog
   module Profiling
@@ -9,11 +9,13 @@ module Datadog
       module HTTP
         # Routes, encodes, and sends tracer data to the trace agent via HTTP.
         class Client < Datadog::Transport::HTTP::Client
-          include Transport::Client
+          def export(flush)
+            send_profiling_flush(flush)
+          end
 
           def send_profiling_flush(flush)
             # Build a request
-            request = Profiling::Transport::Request.new(flush)
+            request = flush
             send_payload(request).tap do |response|
               if response.ok?
                 Datadog.logger.debug('Successfully reported profiling data')

data/lib/datadog/profiling/transport/http/response.rb

@@ -1,7 +1,6 @@
 # typed: true
 
 require 'ddtrace/transport/http/response'
-require 'datadog/profiling/transport/response'
 
 module Datadog
   module Profiling
@@ -11,7 +10,6 @@ module Datadog
         # Response from HTTP transport for profiling
         class Response
           include Datadog::Transport::HTTP::Response
-          include Profiling::Transport::Response
 
           def initialize(http_response, options = {})
             super(http_response)

data/lib/datadog/profiling/transport/http.rb

@@ -80,7 +80,7 @@ module Datadog
         private_class_method def self.configure_for_agentless(transport, profiling_upload_timeout_seconds:, site:, api_key:)
           apis = API.api_defaults
 
-          site_uri = URI(format(Profiling::Ext::Transport::HTTP::URI_TEMPLATE_DD_API, site))
+          site_uri = URI(format(Profiling::OldExt::Transport::HTTP::URI_TEMPLATE_DD_API, site))
           hostname = site_uri.host
           port = site_uri.port
 

data/lib/datadog/profiling.rb

@@ -147,20 +147,20 @@ module Datadog
 
       require 'datadog/profiling/ext/forking'
       require 'datadog/profiling/collectors/code_provenance'
+      require 'datadog/profiling/collectors/cpu_and_wall_time'
       require 'datadog/profiling/collectors/old_stack'
       require 'datadog/profiling/collectors/stack'
       require 'datadog/profiling/stack_recorder'
+      require 'datadog/profiling/old_recorder'
       require 'datadog/profiling/exporter'
-      require 'datadog/profiling/recorder'
       require 'datadog/profiling/scheduler'
       require 'datadog/profiling/tasks/setup'
-      require 'datadog/profiling/transport/io'
-      require 'datadog/profiling/transport/http'
       require 'datadog/profiling/profiler'
       require 'datadog/profiling/native_extension'
       require 'datadog/profiling/trace_identifiers/helper'
       require 'datadog/profiling/pprof/pprof_pb'
       require 'datadog/profiling/tag_builder'
+      require 'datadog/profiling/http_transport'
 
       true
     end