ddtrace 0.48.0 → 0.49.0

data/lib/ddtrace/profiling/profiler.rb

@@ -16,6 +16,8 @@ module Datadog
     end
 
     def shutdown!
+      Datadog.logger.debug('Shutting down profiler')
+
       collectors.each do |collector|
         collector.enabled = false
         collector.stop(true)

data/lib/ddtrace/profiling/recorder.rb

@@ -16,6 +16,9 @@ module Datadog
         event_classes.each do |event_class|
           @buffers[event_class] = Profiling::Buffer.new(max_size)
         end
+
+        # Event classes can only be added ahead of time
+        @buffers.freeze
       end
 
       def [](event_class)
@@ -59,6 +62,11 @@ module Datadog
         )
       end
 
+      # NOTE: Remember that if the recorder is being accessed by multiple threads, this is an inherently racy operation.
+      def empty?
+        @buffers.values.all?(&:empty?)
+      end
+
       # Error when event of an unknown type is used with the Recorder
       class UnknownEventError < StandardError
         attr_reader :event_class

data/lib/ddtrace/profiling/scheduler.rb

@@ -5,31 +5,36 @@ require 'ddtrace/workers/polling'
 
 module Datadog
   module Profiling
-    # Periodically (every DEFAULT_INTERVAL seconds) takes data from the `Recorder` and pushes them to all configured
+    # Periodically (every DEFAULT_INTERVAL_SECONDS) takes data from the `Recorder` and pushes them to all configured
     # `Exporter`s. Runs on its own background thread.
     class Scheduler < Worker
       include Workers::Polling
 
-      DEFAULT_INTERVAL = 60
-      MIN_INTERVAL = 0
+      DEFAULT_INTERVAL_SECONDS = 60
+      MIN_INTERVAL_SECONDS = 0
 
       attr_reader \
         :exporters,
         :recorder
 
-      def initialize(recorder, exporters, options = {})
+      def initialize(
+        recorder,
+        exporters,
+        fork_policy: Workers::Async::Thread::FORK_POLICY_RESTART, # Restart in forks by default
+        interval: DEFAULT_INTERVAL_SECONDS,
+        enabled: true
+      )
         @recorder = recorder
         @exporters = [exporters].flatten
 
         # Workers::Async::Thread settings
-        # Restart in forks by default
-        self.fork_policy = options[:fork_policy] || Workers::Async::Thread::FORK_POLICY_RESTART
+        self.fork_policy = fork_policy
 
         # Workers::IntervalLoop settings
-        self.loop_base_interval = options[:interval] || DEFAULT_INTERVAL
+        self.loop_base_interval = interval
 
         # Workers::Polling settings
-        self.enabled = options.key?(:enabled) ? options[:enabled] == true : true
+        self.enabled = enabled
       end
 
       def start
@@ -51,6 +56,21 @@ module Datadog
         recorder.flush
       end
 
+      # Configure Workers::IntervalLoop to not report immediately when scheduler starts
+      #
+      # When a scheduler gets created (or reset), we don't want it to immediately try to flush; we want it to wait for
+      # the loop wait time first. This avoids an issue where the scheduler reported a mostly-empty profile if the
+      # application just started but this thread took a bit longer so there's already samples in the recorder.
+      def loop_wait_before_first_iteration?
+        true
+      end
+
+      def work_pending?
+        !recorder.empty?
+      end
+
+      private
+
       def flush_and_wait
         run_time = Datadog::Utils::Time.measure do
           flush_events
@@ -58,7 +78,7 @@ module Datadog
 
         # Update wait time to try to wake consistently on time.
         # Don't drop below the minimum interval.
-        self.loop_wait_time = [loop_base_interval - run_time, MIN_INTERVAL].max
+        self.loop_wait_time = [loop_base_interval - run_time, MIN_INTERVAL_SECONDS].max
       end
 
       def flush_events
@@ -71,8 +91,9 @@ module Datadog
             begin
               exporter.export(flush)
             rescue StandardError => e
-              error_details = "Cause: #{e} Location: #{e.backtrace.first}"
-              Datadog.logger.error("Unable to export #{flush.event_count} profiling events. #{error_details}")
+              Datadog.logger.error(
+                "Unable to export #{flush.event_count} profiling events. Cause: #{e} Location: #{e.backtrace.first}"
+              )
             end
           end
         end

data/lib/ddtrace/profiling/transport/http.rb

@@ -13,7 +13,6 @@ require 'ddtrace/transport/http/adapters/unix_socket'
 module Datadog
   module Profiling
     module Transport
-      # TODO: Consolidate with Dataog::Transport::HTTP
       # Namespace for HTTP transport components
       module HTTP
         module_function
@@ -25,31 +24,32 @@ module Datadog
 
         # Builds a new Transport::HTTP::Client with default settings
         # Pass a block to override any settings.
-        def default(options = {})
+        def default(profiling_upload_timeout_seconds:, agent_settings: nil, site: nil, api_key: nil)
           new do |transport|
             transport.headers default_headers
 
             # Configure adapter & API
-            if options[:site] && options[:api_key]
-              configure_for_agentless(transport, options)
+            if site && api_key
+              configure_for_agentless(
+                transport,
+                profiling_upload_timeout_seconds: profiling_upload_timeout_seconds,
+                site: site,
+                api_key: api_key
+              )
             else
-              configure_for_agent(transport, options)
+              unless agent_settings
+                raise(
+                  ArgumentError,
+                  "Missing configuration for #{self}.default: All of `agent_settings`, `site` and `api_key` are nil"
+                )
+              end
+
+              configure_for_agent(
+                transport,
+                profiling_upload_timeout_seconds: profiling_upload_timeout_seconds,
+                agent_settings: agent_settings
+              )
             end
-
-            # Additional options
-            unless options.empty?
-              # Change default API
-              transport.default_api = options[:api_version] if options.key?(:api_version)
-
-              # Add headers
-              transport.headers options[:headers] if options.key?(:headers)
-
-              # Execute on_build callback
-              options[:on_build].call(transport) if options[:on_build].is_a?(Proc)
-            end
-
-            # Call block to apply any customization, if provided.
-            yield(transport) if block_given?
           end
         end
 
@@ -66,56 +66,52 @@ module Datadog
           end
         end
 
-        def default_adapter
+        private_class_method def default_adapter
           :net_http
         end
 
-        def default_hostname
-          ENV.fetch(Datadog::Ext::Transport::HTTP::ENV_DEFAULT_HOST, Datadog::Ext::Transport::HTTP::DEFAULT_HOST)
-        end
-
-        def default_port
-          ENV.fetch(Datadog::Ext::Transport::HTTP::ENV_DEFAULT_PORT, Datadog::Ext::Transport::HTTP::DEFAULT_PORT).to_i
-        end
-
-        def configure_for_agent(transport, options = {})
+        private_class_method def configure_for_agent(transport, profiling_upload_timeout_seconds:, agent_settings:)
           apis = API.agent_defaults
 
-          hostname = options[:hostname] || default_hostname
-          port = options[:port] || default_port
-
-          adapter_options = {}
-          adapter_options[:timeout] = options[:timeout] if options.key?(:timeout)
-          adapter_options[:ssl] = options[:ssl] if options.key?(:ssl)
-
-          transport.adapter default_adapter, hostname, port, adapter_options
-          transport.api API::V1, apis[API::V1], default: true
+          transport.adapter(
+            default_adapter,
+            agent_settings.hostname,
+            agent_settings.port,
+            # We explictly use profiling_upload_timeout_seconds instead of agent_settings.timeout because profile
+            # uploads are bigger and thus we employ a separate configuration.
+            timeout: profiling_upload_timeout_seconds,
+            ssl: agent_settings.ssl
+          )
+          transport.api(API::V1, apis[API::V1], default: true)
+
+          # NOTE: This proc, when it exists, usually overrides the transport specified above
+          if agent_settings.deprecated_for_removal_transport_configuration_proc
+            agent_settings.deprecated_for_removal_transport_configuration_proc.call(transport)
+          end
         end
 
-        def configure_for_agentless(transport, options = {})
+        private_class_method def configure_for_agentless(transport, profiling_upload_timeout_seconds:, site:, api_key:)
           apis = API.api_defaults
 
-          site_uri = URI(format(Datadog::Ext::Profiling::Transport::HTTP::URI_TEMPLATE_DD_API, options[:site]))
-          hostname = options[:hostname] || site_uri.host
-          port = options[:port] || site_uri.port
-
-          adapter_options = {}
-          adapter_options[:timeout] = options[:timeout] if options.key?(:timeout)
-          adapter_options[:ssl] = options[:ssl] || (site_uri.scheme == 'https'.freeze)
-
-          transport.adapter default_adapter, hostname, port, adapter_options
-          transport.api API::V1, apis[API::V1], default: true
-          transport.headers(Datadog::Ext::Transport::HTTP::HEADER_DD_API_KEY => options[:api_key])
+          site_uri = URI(format(Datadog::Ext::Profiling::Transport::HTTP::URI_TEMPLATE_DD_API, site))
+          hostname = site_uri.host
+          port = site_uri.port
+
+          transport.adapter(
+            default_adapter,
+            hostname,
+            port,
+            timeout: profiling_upload_timeout_seconds,
+            ssl: site_uri.scheme == 'https'
+          )
+          transport.api(API::V1, apis[API::V1], default: true)
+          transport.headers(Datadog::Ext::Transport::HTTP::HEADER_DD_API_KEY => api_key)
         end
 
         # Add adapters to registry
         Builder::REGISTRY.set(Datadog::Transport::HTTP::Adapters::Net, :net_http)
         Builder::REGISTRY.set(Datadog::Transport::HTTP::Adapters::Test, :test)
         Builder::REGISTRY.set(Datadog::Transport::HTTP::Adapters::UnixSocket, :unix)
-
-        private \
-          :configure_for_agent,
-          :configure_for_agentless
       end
     end
   end

data/lib/ddtrace/profiling/transport/http/api/endpoint.rb

@@ -48,6 +48,7 @@ module Datadog
                   "#{FORM_FIELD_TAG_RUNTIME_ENGINE}:#{flush.runtime_engine}",
                   "#{FORM_FIELD_TAG_RUNTIME_PLATFORM}:#{flush.runtime_platform}",
                   "#{FORM_FIELD_TAG_RUNTIME_VERSION}:#{flush.runtime_version}",
+                  "#{FORM_FIELD_TAG_PID}:#{Process.pid}",
                   "#{FORM_FIELD_TAG_PROFILER_VERSION}:#{flush.profiler_version}",
                   # NOTE: Redundant w/ 'runtime'; may want to remove this later.
                   "#{FORM_FIELD_TAG_LANGUAGE}:#{flush.language}",

data/lib/ddtrace/runtime/container.rb

@@ -8,9 +8,10 @@ module Datadog
       UUID_PATTERN = '[0-9a-f]{8}[-_]?[0-9a-f]{4}[-_]?[0-9a-f]{4}[-_]?[0-9a-f]{4}[-_]?[0-9a-f]{12}'.freeze
       CONTAINER_PATTERN = '[0-9a-f]{64}'.freeze
 
-      POD_REGEX = /(pod)?(#{UUID_PATTERN})(?:.slice)?$/.freeze
-      CONTAINER_REGEX = /(#{UUID_PATTERN}|#{CONTAINER_PATTERN})(?:.scope)?$/.freeze
-      FARGATE_14_CONTAINER_REGEX = /[0-9a-f]{32}-[0-9]{10}/.freeze
+      PLATFORM_REGEX = /(?<platform>.*?)(?:.slice)?$/.freeze
+      POD_REGEX = /(?<pod>(pod)?#{UUID_PATTERN})(?:.slice)?$/.freeze
+      CONTAINER_REGEX = /(?<container>#{UUID_PATTERN}|#{CONTAINER_PATTERN})(?:.scope)?$/.freeze
+      FARGATE_14_CONTAINER_REGEX = /(?<container>[0-9a-f]{32}-[0-9]{10})/.freeze
 
       Descriptor = Struct.new(
         :platform,
@@ -45,16 +46,23 @@ module Datadog
               parts.shift # Remove leading empty part
 
               # Read info from path
-              platform = parts[0]
+              next if parts.empty?
+
+              platform = parts[0][PLATFORM_REGEX, :platform]
               container_id, task_uid = nil
 
               case parts.length
+              when 0..1
+                next
               when 2
-                container_id = parts[-1][CONTAINER_REGEX] || parts[-1][FARGATE_14_CONTAINER_REGEX]
+                container_id = parts[-1][CONTAINER_REGEX, :container] \
+                               || parts[-1][FARGATE_14_CONTAINER_REGEX, :container]
               else
-                platform = parts[0]
-                container_id = parts[-1][CONTAINER_REGEX]
-                task_uid = parts[-2][POD_REGEX]
+                if (container_id = parts[-1][CONTAINER_REGEX, :container])
+                  task_uid = parts[-2][POD_REGEX, :pod]
+                else
+                  container_id = parts[-1][FARGATE_14_CONTAINER_REGEX, :container]
+                end
               end
 
               # If container ID wasn't found, ignore.

data/lib/ddtrace/tracer.rb

@@ -85,7 +85,7 @@ module Datadog
       @provider = options.fetch(:context_provider, Datadog::DefaultContextProvider.new)
       @sampler = options.fetch(:sampler, Datadog::AllSampler.new)
       @tags = options.fetch(:tags, {})
-      @writer = options.fetch(:writer, Datadog::Writer.new)
+      @writer = options.fetch(:writer) { Datadog::Writer.new }
 
       # Instance variables
       @mutex = Mutex.new
@@ -391,20 +391,14 @@ module Datadog
 
     # TODO: Move this kind of configuration building out of the tracer.
     #       Tracer should not have this kind of knowledge of writer.
-    # rubocop:disable Metrics/PerceivedComplexity
-    # rubocop:disable Metrics/CyclomaticComplexity
-    # rubocop:disable Metrics/MethodLength
     def configure_writer(options = {})
-      hostname = options.fetch(:hostname, nil)
-      port = options.fetch(:port, nil)
       sampler = options.fetch(:sampler, nil)
       priority_sampling = options.fetch(:priority_sampling, nil)
       writer = options.fetch(:writer, nil)
-      transport_options = options.fetch(:transport_options, {}).dup
+      agent_settings = options.fetch(:agent_settings, nil)
 
       # Compile writer options
       writer_options = options.fetch(:writer_options, {}).dup
-      rebuild_writer = !writer_options.empty?
 
       # Re-build the sampler and writer if priority sampling is enabled,
       # but neither are configured. Verify the sampler isn't already a
@@ -417,35 +411,20 @@ module Datadog
         end
       elsif priority_sampling != false && !@sampler.is_a?(PrioritySampler)
         writer_options[:priority_sampler] = activate_priority_sampling!(@sampler)
-        rebuild_writer = true
       elsif priority_sampling == false
         deactivate_priority_sampling!(sampler)
-        rebuild_writer = true
       elsif @sampler.is_a?(PrioritySampler)
         # Make sure to add sampler to options if transport is rebuilt.
         writer_options[:priority_sampler] = @sampler
       end
 
-      # Apply options to transport
-      if transport_options.is_a?(Proc)
-        transport_options = { on_build: transport_options }
-        rebuild_writer = true
-      end
-
-      if hostname || port
-        transport_options[:hostname] = hostname unless hostname.nil?
-        transport_options[:port] = port unless port.nil?
-        rebuild_writer = true
-      end
+      writer_options[:agent_settings] = agent_settings if agent_settings
 
-      writer_options[:transport_options] = transport_options
+      # Make sure old writer is shut down before throwing away.
+      # Don't want additional threads running...
+      @writer.stop unless writer.nil?
 
-      if rebuild_writer || writer
-        # Make sure old writer is shut down before throwing away.
-        # Don't want additional threads running...
-        @writer.stop unless writer.nil?
-        @writer = writer || Writer.new(writer_options)
-      end
+      @writer = writer || Writer.new(writer_options)
     end
 
     def activate_priority_sampling!(base_sampler = nil)

data/lib/ddtrace/transport/http.rb

@@ -25,11 +25,23 @@ module Datadog
 
       # Builds a new Transport::HTTP::Client with default settings
       # Pass a block to override any settings.
-      def default(options = {})
+      def default(agent_settings: Datadog::Configuration::AgentSettingsResolver::ENVIRONMENT_AGENT_SETTINGS, **options)
         new do |transport|
-          transport.adapter default_adapter, default_hostname, default_port, timeout: 1
+          transport.adapter(
+            default_adapter,
+            agent_settings.hostname,
+            agent_settings.port,
+            timeout: agent_settings.timeout_seconds,
+            ssl: agent_settings.ssl
+          )
           transport.headers default_headers
 
+          if agent_settings.deprecated_for_removal_transport_configuration_options
+            # The deprecated_for_removal_transport_configuration_options take precedence over any options the caller
+            # specifies
+            options = options.merge(**agent_settings.deprecated_for_removal_transport_configuration_options)
+          end
+
           apis = API.defaults
 
           transport.api API::V4, apis[API::V4], fallback: API::V3, default: true
@@ -38,29 +50,15 @@ module Datadog
 
           # Apply any settings given by options
           unless options.empty?
-            # Change hostname/port
-            if [:hostname, :port, :timeout, :ssl].any? { |key| options.key?(key) }
-              hostname = options[:hostname] || default_hostname
-              port = options[:port] || default_port
-
-              adapter_options = { timeout: 1 }
-              adapter_options[:timeout] = options[:timeout] if options.key?(:timeout)
-              adapter_options[:ssl] = options[:ssl] if options.key?(:ssl)
-
-              transport.adapter default_adapter, hostname, port, adapter_options
-            end
-
-            # Change default API
             transport.default_api = options[:api_version] if options.key?(:api_version)
-
-            # Add headers
             transport.headers options[:headers] if options.key?(:headers)
+          end
 
-            # Execute on_build callback
-            options[:on_build].call(transport) if options[:on_build].is_a?(Proc)
+          if agent_settings.deprecated_for_removal_transport_configuration_proc
+            agent_settings.deprecated_for_removal_transport_configuration_proc.call(transport)
           end
 
-          # Call block to apply any customization, if provided.
+          # Call block to apply any customization, if provided
           yield(transport) if block_given?
         end
       end
@@ -82,26 +80,31 @@ module Datadog
         :net_http
       end
 
-      def default_hostname
-        return default_url.hostname if default_url
+      def default_hostname(logger: Datadog.logger)
+        logger.warn(
+          'Deprecated for removal: Using #default_hostname for configuration is deprecated and will ' \
+          'be removed on a future ddtrace release.'
+        )
 
-        ENV.fetch(Datadog::Ext::Transport::HTTP::ENV_DEFAULT_HOST, Datadog::Ext::Transport::HTTP::DEFAULT_HOST)
+        Datadog::Configuration::AgentSettingsResolver::ENVIRONMENT_AGENT_SETTINGS.hostname
       end
 
-      def default_port
-        return default_url.port if default_url
+      def default_port(logger: Datadog.logger)
+        logger.warn(
+          'Deprecated for removal: Using #default_hostname for configuration is deprecated and will ' \
+          'be removed on a future ddtrace release.'
+        )
 
-        ENV.fetch(Datadog::Ext::Transport::HTTP::ENV_DEFAULT_PORT, Datadog::Ext::Transport::HTTP::DEFAULT_PORT).to_i
+        Datadog::Configuration::AgentSettingsResolver::ENVIRONMENT_AGENT_SETTINGS.port
       end
 
-      def default_url
-        url_env = ENV.fetch(Datadog::Ext::Transport::HTTP::ENV_DEFAULT_URL, nil)
+      def default_url(logger: Datadog.logger)
+        logger.warn(
+          'Deprecated for removal: Using #default_url for configuration is deprecated and will ' \
+          'be removed on a future ddtrace release.'
+        )
 
-        if url_env
-          uri_parsed = URI.parse(url_env)
-
-          uri_parsed if %w[http https].include?(uri_parsed.scheme)
-        end
+        nil
       end
 
       # Add adapters to registry