ddtrace 1.1.0 → 1.4.0

data/lib/datadog/profiling/pprof/stack_sample.rb

@@ -1,9 +1,9 @@
 # typed: true
 
-require 'datadog/profiling/ext'
-require 'datadog/profiling/events/stack'
-require 'datadog/profiling/pprof/builder'
-require 'datadog/profiling/pprof/converter'
+require_relative '../ext'
+require_relative '../events/stack'
+require_relative 'builder'
+require_relative 'converter'
 
 module Datadog
   module Profiling

data/lib/datadog/profiling/pprof/string_table.rb

@@ -1,6 +1,6 @@
 # typed: strict
 
-require 'datadog/core/utils/string_table'
+require_relative '../../core/utils/string_table'
 
 module Datadog
   module Profiling

data/lib/datadog/profiling/pprof/template.rb

@@ -1,11 +1,11 @@
 # typed: false
 
-require 'datadog/profiling/pprof/payload'
-require 'datadog/profiling/pprof/message_set'
-require 'datadog/profiling/pprof/builder'
+require_relative 'payload'
+require_relative 'message_set'
+require_relative 'builder'
 
-require 'datadog/profiling/events/stack'
-require 'datadog/profiling/pprof/stack_sample'
+require_relative '../events/stack'
+require_relative 'stack_sample'
 
 module Datadog
   module Profiling

data/lib/datadog/profiling/preload.rb

@@ -1,5 +1,5 @@
 # typed: strict
 
-require 'ddtrace'
+require_relative '../../ddtrace'
 
 Datadog::Profiling.start_if_enabled

data/lib/datadog/profiling/scheduler.rb

@@ -1,42 +1,41 @@
 # typed: true
 
-require 'datadog/core/utils/time'
+require_relative '../core/utils/time'
 
-require 'datadog/core/worker'
-require 'datadog/core/workers/polling'
+require_relative '../core/worker'
+require_relative '../core/workers/polling'
 
 module Datadog
   module Profiling
-    # Periodically (every DEFAULT_INTERVAL_SECONDS) takes data from the `Recorder` and pushes them to all configured
-    # `Exporter`s. Runs on its own background thread.
+    # Periodically (every DEFAULT_INTERVAL_SECONDS) takes a profile from the `Exporter` and reports it using the
+    # configured transport. Runs on its own background thread.
     class Scheduler < Core::Worker
       include Core::Workers::Polling
 
       DEFAULT_INTERVAL_SECONDS = 60
       MINIMUM_INTERVAL_SECONDS = 0
 
-      # Profiles with duration less than this will not be reported
-      PROFILE_DURATION_THRESHOLD_SECONDS = 1
-
       # We sleep for at most this duration seconds before reporting data to avoid multi-process applications all
       # reporting profiles at the exact same time
       DEFAULT_FLUSH_JITTER_MAXIMUM_SECONDS = 3
 
-      private_constant :DEFAULT_INTERVAL_SECONDS, :MINIMUM_INTERVAL_SECONDS, :PROFILE_DURATION_THRESHOLD_SECONDS
+      private
 
       attr_reader \
-        :exporters,
-        :recorder
+        :exporter,
+        :transport
+
+      public
 
       def initialize(
-        recorder,
-        exporters,
+        exporter:,
+        transport:,
         fork_policy: Core::Workers::Async::Thread::FORK_POLICY_RESTART, # Restart in forks by default
         interval: DEFAULT_INTERVAL_SECONDS,
         enabled: true
       )
-        @recorder = recorder
-        @exporters = [exporters].flatten
+        @exporter = exporter
+        @transport = transport
 
         # Workers::Async::Thread settings
         self.fork_policy = fork_policy
@@ -68,23 +67,23 @@ module Datadog
       end
 
       def after_fork
-        # Clear recorder's buffers by flushing events.
-        # Objects from parent process will copy-on-write,
-        # and we don't want to send events for the wrong process.
-        recorder.flush
+        # Clear any existing profiling state.
+        # We don't want the child process to report profiling data from its parent.
+        Datadog.logger.debug('Flushing exporter in child process #after_fork and discarding data')
+        exporter.flush
       end
 
       # Configure Workers::IntervalLoop to not report immediately when scheduler starts
       #
       # When a scheduler gets created (or reset), we don't want it to immediately try to flush; we want it to wait for
       # the loop wait time first. This avoids an issue where the scheduler reported a mostly-empty profile if the
-      # application just started but this thread took a bit longer so there's already samples in the recorder.
+      # application just started but this thread took a bit longer so there's already profiling data in the exporter.
       def loop_wait_before_first_iteration?
         true
       end
 
       def work_pending?
-        !recorder.empty?
+        exporter.can_flush?
       end
 
       private
@@ -100,16 +99,10 @@ module Datadog
       end
 
       def flush_events
-        # Get events from recorder
-        flush = recorder.flush
+        # Collect data to be exported
+        flush = exporter.flush
 
-        if duration_below_threshold?(flush)
-          Datadog.logger.debug do
-            "Skipped exporting profiling events as profile duration is below minimum (#{flush.event_count} events skipped)"
-          end
-
-          return flush
-        end
+        return false unless flush
 
         # Sleep for a bit to cause misalignment between profilers in multi-process applications
         #
@@ -127,24 +120,13 @@ module Datadog
           sleep(jitter_seconds)
         end
 
-        # Send events to each exporter
-        if flush.event_count > 0
-          exporters.each do |exporter|
-            begin
-              exporter.export(flush)
-            rescue StandardError => e
-              Datadog.logger.error(
-                "Unable to export #{flush.event_count} profiling events. Cause: #{e} Location: #{Array(e.backtrace).first}"
-              )
-            end
-          end
+        begin
+          transport.export(flush)
+        rescue StandardError => e
+          Datadog.logger.error("Unable to report profile. Cause: #{e} Location: #{Array(e.backtrace).first}")
         end
 
-        flush
-      end
-
-      def duration_below_threshold?(flush)
-        (flush.finish - flush.start) < PROFILE_DURATION_THRESHOLD_SECONDS
+        true
       end
     end
   end

data/lib/datadog/profiling/stack_recorder.rb

@@ -2,11 +2,22 @@
 
 module Datadog
   module Profiling
-    # Used to wrap a ddprof_ffi_Profile in a Ruby object and expose Ruby-level serialization APIs
+    # Stores stack samples in a native libdatadog data structure and expose Ruby-level serialization APIs
+    # Note that `record_sample` is only accessible from native code.
     # Methods prefixed with _native_ are implemented in `stack_recorder.c`
     class StackRecorder
+      def initialize
+        # This mutex works in addition to the fancy C-level mutexes we have in the native side (see the docs there).
+        # It prevents multiple Ruby threads calling serialize at the same time -- something like
+        # `10.times { Thread.new { stack_recorder.serialize } }`.
+        # This isn't something we expect to happen normally, but because it would break the assumptions of the
+        # C-level mutexes (that there is a single serializer thread), we add it here as an extra safeguard against it
+        # accidentally happening.
+        @no_concurrent_synchronize_mutex = Mutex.new
+      end
+
       def serialize
-        status, result = self.class._native_serialize(self)
+        status, result = @no_concurrent_synchronize_mutex.synchronize { self.class._native_serialize(self) }
 
         if status == :ok
           start, finish, encoded_pprof = result
@@ -23,8 +34,7 @@ module Datadog
         end
       end
 
-      # Used only for Ruby 2.2 and below which don't have the native `rb_time_timespec_new` API
-      # Called from native code
+      # Used only for Ruby 2.2 which doesn't have the native `rb_time_timespec_new` API; called from native code
       def self.ruby_time_from(timespec_seconds, timespec_nanoseconds)
         Time.at(0).utc + timespec_seconds + (timespec_nanoseconds.to_r / 1_000_000_000)
       end

data/lib/datadog/profiling/tag_builder.rb

@@ -1,5 +1,7 @@
 # typed: true
 
+require_relative '../core/utils'
+
 module Datadog
   module Profiling
     # Builds a hash of default plus user tags to be included in a profile
@@ -41,7 +43,10 @@ module Datadog
         tags[FORM_FIELD_TAG_SERVICE] = service if service
         tags[FORM_FIELD_TAG_VERSION] = version if version
 
-        user_tags.merge(tags)
+        # Make sure everything is an utf-8 string, to avoid encoding issues in native code/libddprof/further downstream
+        user_tags.merge(tags).map do |key, value|
+          [Datadog::Core::Utils.utf8_encode(key), Datadog::Core::Utils.utf8_encode(value)]
+        end.to_h
       end
     end
   end

data/lib/datadog/profiling/tasks/setup.rb

@@ -1,7 +1,7 @@
 # typed: false
 
-require 'datadog/core/utils/only_once'
-require 'datadog/profiling/ext/forking'
+require_relative '../../core/utils/only_once'
+require_relative '../ext/forking'
 
 module Datadog
   module Profiling

data/lib/datadog/profiling/trace_identifiers/ddtrace.rb

@@ -2,8 +2,8 @@
 
 # typed: true
 
-require 'datadog/tracing'
-require 'datadog/tracing/metadata/ext'
+require_relative '../../tracing'
+require_relative '../../tracing/metadata/ext'
 
 module Datadog
   module Profiling

data/lib/datadog/profiling/trace_identifiers/helper.rb

@@ -2,7 +2,7 @@
 
 # typed: true
 
-require 'datadog/profiling/trace_identifiers/ddtrace'
+require_relative 'ddtrace'
 
 module Datadog
   module Profiling

data/lib/datadog/profiling/transport/http/api/endpoint.rb

@@ -1,10 +1,10 @@
 # typed: true
 
-require 'datadog/core/utils/compression'
-require 'datadog/core/vendor/multipart-post/multipart/post/composite_read_io'
-require 'datadog/profiling/ext'
-require 'datadog/profiling/transport/http/response'
-require 'ddtrace/transport/http/api/endpoint'
+require_relative '../../../../core/utils/compression'
+require_relative '../../../../core/vendor/multipart-post/multipart/post/composite_read_io'
+require_relative '../../../old_ext'
+require_relative '../response'
+require_relative '../../../../../ddtrace/transport/http/api/endpoint'
 
 module Datadog
   module Profiling
@@ -13,7 +13,7 @@ module Datadog
         module API
           # Datadog API endpoint for profiling
           class Endpoint < Datadog::Transport::HTTP::API::Endpoint
-            include Profiling::Ext::Transport::HTTP
+            include Profiling::OldExt::Transport::HTTP
 
             # These tags are read from the flush object (see below) directly and so we ignore any extra copies that
             # may come in the tags hash to avoid duplicates.
@@ -23,7 +23,7 @@ module Datadog
             attr_reader \
               :encoder
 
-            def initialize(path, encoder)
+            def initialize(path, encoder = nil)
               super(:post, path)
               @encoder = encoder
             end
@@ -40,48 +40,26 @@ module Datadog
             end
 
             def build_form(env)
-              flush = env.request.parcel.data
+              flush = env.request
               pprof_file = build_pprof(flush)
 
               form = {
                 FORM_FIELD_INTAKE_VERSION => '3', # Aka 1.3 intake format
                 FORM_FIELD_RECORDING_START => flush.start.utc.iso8601,
                 FORM_FIELD_RECORDING_END => flush.finish.utc.iso8601,
-                FORM_FIELD_TAGS => [
-                  "#{FORM_FIELD_TAG_RUNTIME}:#{flush.language}",
-                  "#{FORM_FIELD_TAG_RUNTIME_ID}:#{flush.runtime_id}",
-                  "#{FORM_FIELD_TAG_RUNTIME_ENGINE}:#{flush.runtime_engine}",
-                  "#{FORM_FIELD_TAG_RUNTIME_PLATFORM}:#{flush.runtime_platform}",
-                  "#{FORM_FIELD_TAG_RUNTIME_VERSION}:#{flush.runtime_version}",
-                  "#{FORM_FIELD_TAG_PID}:#{Process.pid}",
-                  "#{FORM_FIELD_TAG_PROFILER_VERSION}:#{flush.profiler_version}",
-                  # NOTE: Redundant w/ 'runtime'; may want to remove this later.
-                  "#{FORM_FIELD_TAG_LANGUAGE}:#{flush.language}",
-                  "#{FORM_FIELD_TAG_HOST}:#{flush.host}",
-                  *flush
-                    .tags
-                    .reject { |tag_key| TAGS_TO_IGNORE_IN_TAGS_HASH.include?(tag_key) }
-                    .map { |tag_key, tag_value| "#{tag_key}:#{tag_value}" }
-                ],
+                FORM_FIELD_TAGS => flush.tags_as_array.map { |key, value| "#{key}:#{value}" },
                 FORM_FIELD_PPROF_DATA => pprof_file,
-                FORM_FIELD_FAMILY => flush.language,
+                FORM_FIELD_FAMILY => 'ruby',
               }
 
-              # Optional fields
-              form[FORM_FIELD_TAGS] << "#{FORM_FIELD_TAG_SERVICE}:#{flush.service}" unless flush.service.nil?
-              form[FORM_FIELD_TAGS] << "#{FORM_FIELD_TAG_ENV}:#{flush.env}" unless flush.env.nil?
-              form[FORM_FIELD_TAGS] << "#{FORM_FIELD_TAG_VERSION}:#{flush.version}" unless flush.version.nil?
-
               # May not be available/enabled
-              form[FORM_FIELD_CODE_PROVENANCE_DATA] = build_code_provenance(flush) if flush.code_provenance
+              form[FORM_FIELD_CODE_PROVENANCE_DATA] = build_code_provenance(flush) if flush.code_provenance_data
 
               form
             end
 
             def build_pprof(flush)
-              pprof = encoder.encode(flush)
-
-              gzipped_pprof_data = Core::Utils::Compression.gzip(pprof.data)
+              gzipped_pprof_data = flush.pprof_data
 
               Core::Vendor::Multipart::Post::UploadIO.new(
                 StringIO.new(gzipped_pprof_data),
@@ -91,7 +69,7 @@ module Datadog
             end
 
             def build_code_provenance(flush)
-              gzipped_code_provenance = Core::Utils::Compression.gzip(flush.code_provenance)
+              gzipped_code_provenance = flush.code_provenance_data
 
               Core::Vendor::Multipart::Post::UploadIO.new(
                 StringIO.new(gzipped_code_provenance),

data/lib/datadog/profiling/transport/http/api/instance.rb

@@ -1,7 +1,7 @@
 # typed: true
 
-require 'ddtrace/transport/http/api/instance'
-require 'datadog/profiling/transport/http/api/spec'
+require_relative '../../../../../ddtrace/transport/http/api/instance'
+require_relative 'spec'
 
 module Datadog
   module Profiling

data/lib/datadog/profiling/transport/http/api/spec.rb

@@ -1,6 +1,6 @@
 # typed: true
 
-require 'ddtrace/transport/http/api/spec'
+require_relative '../../../../../ddtrace/transport/http/api/spec'
 
 module Datadog
   module Profiling

data/lib/datadog/profiling/transport/http/api.rb

@@ -1,10 +1,10 @@
 # typed: true
 
-require 'ddtrace/transport/http/api/map'
-require 'datadog/profiling/encoding/profile'
-require 'datadog/profiling/transport/http/api/spec'
-require 'datadog/profiling/transport/http/api/instance'
-require 'datadog/profiling/transport/http/api/endpoint'
+require_relative '../../../../ddtrace/transport/http/api/map'
+require_relative '../../encoding/profile'
+require_relative 'api/spec'
+require_relative 'api/instance'
+require_relative 'api/endpoint'
 
 module Datadog
   module Profiling

data/lib/datadog/profiling/transport/http/builder.rb

@@ -1,9 +1,9 @@
 # typed: true
 
-require 'ddtrace/transport/http/builder'
+require_relative '../../../../ddtrace/transport/http/builder'
 
-require 'datadog/profiling/transport/http/api'
-require 'datadog/profiling/transport/http/client'
+require_relative 'api'
+require_relative 'client'
 
 module Datadog
   module Profiling

data/lib/datadog/profiling/transport/http/client.rb

@@ -1,7 +1,7 @@
 # typed: true
 
-require 'ddtrace/transport/http/client'
-require 'datadog/profiling/transport/client'
+require_relative '../../../../ddtrace/transport/http/client'
+require_relative '../../../../ddtrace/transport/request'
 
 module Datadog
   module Profiling
@@ -9,11 +9,13 @@ module Datadog
       module HTTP
         # Routes, encodes, and sends tracer data to the trace agent via HTTP.
         class Client < Datadog::Transport::HTTP::Client
-          include Transport::Client
+          def export(flush)
+            send_profiling_flush(flush)
+          end
 
           def send_profiling_flush(flush)
             # Build a request
-            request = Profiling::Transport::Request.new(flush)
+            request = flush
             send_payload(request).tap do |response|
               if response.ok?
                 Datadog.logger.debug('Successfully reported profiling data')

data/lib/datadog/profiling/transport/http/response.rb

@@ -1,7 +1,6 @@
 # typed: true
 
-require 'ddtrace/transport/http/response'
-require 'datadog/profiling/transport/response'
+require_relative '../../../../ddtrace/transport/http/response'
 
 module Datadog
   module Profiling
@@ -11,7 +10,6 @@ module Datadog
         # Response from HTTP transport for profiling
         class Response
           include Datadog::Transport::HTTP::Response
-          include Profiling::Transport::Response
 
           def initialize(http_response, options = {})
             super(http_response)

data/lib/datadog/profiling/transport/http.rb

@@ -1,17 +1,17 @@
 # typed: true
 
-require 'datadog/core/environment/ext'
-require 'ddtrace/transport/ext'
+require_relative '../../core/environment/ext'
+require_relative '../../../ddtrace/transport/ext'
 
-require 'datadog/core/environment/container'
-require 'datadog/core/environment/variable_helpers'
+require_relative '../../core/environment/container'
+require_relative '../../core/environment/variable_helpers'
 
-require 'datadog/profiling/transport/http/builder'
-require 'datadog/profiling/transport/http/api'
+require_relative 'http/builder'
+require_relative 'http/api'
 
-require 'ddtrace/transport/http/adapters/net'
-require 'ddtrace/transport/http/adapters/test'
-require 'ddtrace/transport/http/adapters/unix_socket'
+require_relative '../../../ddtrace/transport/http/adapters/net'
+require_relative '../../../ddtrace/transport/http/adapters/test'
+require_relative '../../../ddtrace/transport/http/adapters/unix_socket'
 
 module Datadog
   module Profiling
@@ -80,7 +80,7 @@ module Datadog
         private_class_method def self.configure_for_agentless(transport, profiling_upload_timeout_seconds:, site:, api_key:)
           apis = API.api_defaults
 
-          site_uri = URI(format(Profiling::Ext::Transport::HTTP::URI_TEMPLATE_DD_API, site))
+          site_uri = URI(format(Profiling::OldExt::Transport::HTTP::URI_TEMPLATE_DD_API, site))
           hostname = site_uri.host
           port = site_uri.port
 
@@ -100,12 +100,18 @@ module Datadog
         end
 
         # Add adapters to registry
-        Datadog::Transport::HTTP::Builder::REGISTRY.set(Datadog::Transport::HTTP::Adapters::Net,
-                                                        Datadog::Transport::Ext::HTTP::ADAPTER)
-        Datadog::Transport::HTTP::Builder::REGISTRY.set(Datadog::Transport::HTTP::Adapters::Test,
-                                                        Datadog::Transport::Ext::Test::ADAPTER)
-        Datadog::Transport::HTTP::Builder::REGISTRY.set(Datadog::Transport::HTTP::Adapters::UnixSocket,
-                                                        Datadog::Transport::Ext::UnixSocket::ADAPTER)
+        Datadog::Transport::HTTP::Builder::REGISTRY.set(
+          Datadog::Transport::HTTP::Adapters::Net,
+          Datadog::Transport::Ext::HTTP::ADAPTER
+        )
+        Datadog::Transport::HTTP::Builder::REGISTRY.set(
+          Datadog::Transport::HTTP::Adapters::Test,
+          Datadog::Transport::Ext::Test::ADAPTER
+        )
+        Datadog::Transport::HTTP::Builder::REGISTRY.set(
+          Datadog::Transport::HTTP::Adapters::UnixSocket,
+          Datadog::Transport::Ext::UnixSocket::ADAPTER
+        )
       end
     end
   end

data/lib/datadog/profiling.rb

@@ -1,8 +1,8 @@
 # typed: true
 
-require 'datadog/core'
-require 'datadog/core/environment/variable_helpers'
-require 'datadog/core/utils/only_once'
+require_relative 'core'
+require_relative 'core/environment/variable_helpers'
+require_relative 'core/utils/only_once'
 
 module Datadog
   # Contains profiler for generating stack profiles, etc.
@@ -132,7 +132,7 @@ module Datadog
 
     private_class_method def self.try_loading_native_library
       begin
-        require 'datadog/profiling/load_native_extension'
+        require_relative 'profiling/load_native_extension'
 
         success =
           defined?(Profiling::NativeExtension) && Profiling::NativeExtension.send(:native_working?)
@@ -145,22 +145,23 @@ module Datadog
     private_class_method def self.load_profiling
       return false unless supported?
 
-      require 'datadog/profiling/ext/forking'
-      require 'datadog/profiling/collectors/code_provenance'
-      require 'datadog/profiling/collectors/old_stack'
-      require 'datadog/profiling/collectors/stack'
-      require 'datadog/profiling/stack_recorder'
-      require 'datadog/profiling/exporter'
-      require 'datadog/profiling/recorder'
-      require 'datadog/profiling/scheduler'
-      require 'datadog/profiling/tasks/setup'
-      require 'datadog/profiling/transport/io'
-      require 'datadog/profiling/transport/http'
-      require 'datadog/profiling/profiler'
-      require 'datadog/profiling/native_extension'
-      require 'datadog/profiling/trace_identifiers/helper'
-      require 'datadog/profiling/pprof/pprof_pb'
-      require 'datadog/profiling/tag_builder'
+      require_relative 'profiling/ext/forking'
+      require_relative 'profiling/collectors/code_provenance'
+      require_relative 'profiling/collectors/cpu_and_wall_time'
+      require_relative 'profiling/collectors/cpu_and_wall_time_worker'
+      require_relative 'profiling/collectors/old_stack'
+      require_relative 'profiling/collectors/stack'
+      require_relative 'profiling/stack_recorder'
+      require_relative 'profiling/old_recorder'
+      require_relative 'profiling/exporter'
+      require_relative 'profiling/scheduler'
+      require_relative 'profiling/tasks/setup'
+      require_relative 'profiling/profiler'
+      require_relative 'profiling/native_extension'
+      require_relative 'profiling/trace_identifiers/helper'
+      require_relative 'profiling/pprof/pprof_pb'
+      require_relative 'profiling/tag_builder'
+      require_relative 'profiling/http_transport'
 
       true
     end

data/lib/datadog/tracing/analytics.rb

@@ -1,6 +1,6 @@
 # typed: true
 
-require 'datadog/tracing/metadata/ext'
+require_relative 'metadata/ext'
 
 module Datadog
   module Tracing

data/lib/datadog/tracing/buffer.rb

@@ -1,10 +1,10 @@
 # typed: true
 
-require 'datadog/core'
-require 'datadog/core/environment/ext'
-require 'datadog/core/buffer/thread_safe'
-require 'datadog/core/buffer/cruby'
-require 'datadog/core/diagnostics/health'
+require_relative '../core'
+require_relative '../core/environment/ext'
+require_relative '../core/buffer/thread_safe'
+require_relative '../core/buffer/cruby'
+require_relative '../core/diagnostics/health'
 
 module Datadog
   module Tracing

data/lib/datadog/tracing/context.rb

@@ -1,6 +1,6 @@
 # typed: true
 
-require 'datadog/core/utils/forking'
+require_relative '../core/utils/forking'
 
 module Datadog
   module Tracing