ddtrace 0.46.0 → 0.51.0

data/lib/ddtrace/configuration/base.rb

@@ -1,4 +1,4 @@
-require 'ddtrace/environment'
+require 'datadog/core/environment/variable_helpers'
 require 'ddtrace/configuration/options'
 
 module Datadog
@@ -6,12 +6,12 @@ module Datadog
     # Basic configuration behavior
     module Base
       def self.included(base)
-        base.send(:extend, Datadog::Environment::Helpers)
-        base.send(:include, Datadog::Environment::Helpers)
-        base.send(:include, Options)
+        base.extend(Datadog::Core::Environment::VariableHelpers)
+        base.include(Datadog::Core::Environment::VariableHelpers)
+        base.include(Options)
 
-        base.send(:extend, ClassMethods)
-        base.send(:include, InstanceMethods)
+        base.extend(ClassMethods)
+        base.include(InstanceMethods)
       end
 
       # Class methods for configuration
@@ -53,10 +53,7 @@ module Datadog
           ordering = self.class.options.dependency_order
           sorted_opts = opts.sort_by do |name, _value|
             ordering.index(name) || (ordering.length + 1)
-          end
-
-          # Ruby 2.0 doesn't support Array#to_h
-          sorted_opts = Hash[*sorted_opts.flatten]
+          end.to_h
 
           # Apply options in sort order
           sorted_opts.each do |name, value|

data/lib/ddtrace/configuration/components.rb

@@ -1,12 +1,16 @@
+require 'ddtrace/configuration/agent_settings_resolver'
 require 'ddtrace/diagnostics/health'
 require 'ddtrace/logger'
+require 'ddtrace/profiling'
 require 'ddtrace/runtime/metrics'
 require 'ddtrace/tracer'
+require 'ddtrace/sync_writer'
 require 'ddtrace/workers/runtime_metrics'
 
 module Datadog
   module Configuration
     # Global components for the trace library.
+    # rubocop:disable Metrics/ClassLength
     # rubocop:disable Layout/LineLength
     class Components
       class << self
@@ -15,7 +19,7 @@ module Datadog
           options = { enabled: settings.enabled }
           options[:statsd] = settings.statsd unless settings.statsd.nil?
 
-          Datadog::Diagnostics::Health::Metrics.new(options)
+          Datadog::Diagnostics::Health::Metrics.new(**options)
         end
 
         def build_logger(settings)
@@ -30,7 +34,7 @@ module Datadog
           options[:statsd] = settings.runtime_metrics.statsd unless settings.runtime_metrics.statsd.nil?
           options[:services] = [settings.service] unless settings.service.nil?
 
-          Datadog::Runtime::Metrics.new(options)
+          Datadog::Runtime::Metrics.new(**options)
         end
 
         def build_runtime_metrics_worker(settings)
@@ -43,7 +47,7 @@ module Datadog
           Datadog::Workers::RuntimeMetrics.new(options)
         end
 
-        def build_tracer(settings)
+        def build_tracer(settings, agent_settings)
           # If a custom tracer has been provided, use it instead.
           # Ignore all other options (they should already be configured.)
           tracer = settings.tracer.instance
@@ -61,40 +65,137 @@ module Datadog
           #       tracer initialization for now. Just reconfigure using the
           #       existing mutable #configure function. Remove when these components
           #       are extracted.
-          tracer.configure(build_tracer_options(settings))
+          tracer.configure(agent_settings: agent_settings, **build_tracer_options(settings, agent_settings))
 
           tracer
         end
 
+        def build_profiler(settings, agent_settings)
+          return unless Datadog::Profiling.supported? && settings.profiling.enabled
+
+          unless defined?(Datadog::Profiling::Tasks::Setup)
+            # In #1545 a user reported a NameError due to this constant being uninitialized
+            # I've documented my suspicion on why that happened in
+            # https://github.com/DataDog/dd-trace-rb/issues/1545#issuecomment-856049025
+            #
+            # > Thanks for the info! It seems to feed into my theory: there's two moments in the code where we check if
+            # > profiler is "supported": 1) when loading ddtrace (inside preload) and 2) when starting the profile
+            # > after Datadog.configure gets run.
+            # > The problem is that the code assumes that both checks 1) and 2) will always reach the same conclusion:
+            # > either profiler is supported, or profiler is not supported.
+            # > In the problematic case, it looks like in your case check 1 decides that profiler is not
+            # > supported => doesn't load it, and then check 2 decides that it is => assumes it is loaded and tries to
+            # > start it.
+            #
+            # I was never able to validate if this was the issue or why exactly .supported? would change its mind BUT
+            # just in case it happens again, I've left this check which avoids breaking the user's application AND
+            # would instead direct them to report it to us instead, so that we can investigate what's wrong.
+            #
+            # TODO: As of June 2021, most checks in .supported? are related to the google-protobuf gem; so it's
+            # very likely that it was the origin of the issue we saw. Thus, if, as planned we end up moving away from
+            # protobuf OR enough time has passed and no users saw the issue again, we can remove this check altogether.
+            Datadog.logger.error(
+              'Profiling was marked as supported and enabled, but setup task was not loaded properly. ' \
+              'Please report this at https://github.com/DataDog/dd-trace-rb/blob/master/CONTRIBUTING.md#found-a-bug'
+            )
+
+            return
+          end
+
+          # Load extensions needed to support some of the Profiling features
+          Datadog::Profiling::Tasks::Setup.new.run
+
+          # NOTE: Please update the Initialization section of ProfilingDevelopment.md with any changes to this method
+
+          recorder = build_profiler_recorder(settings)
+          collectors = build_profiler_collectors(settings, recorder)
+          exporters = build_profiler_exporters(settings, agent_settings)
+          scheduler = build_profiler_scheduler(settings, recorder, exporters)
+
+          Datadog::Profiler.new(collectors, scheduler)
+        end
+
         private
 
         def build_tracer_tags(settings)
           settings.tags.dup.tap do |tags|
-            tags['env'] = settings.env unless settings.env.nil?
-            tags['version'] = settings.version unless settings.version.nil?
+            tags[Ext::Environment::TAG_ENV] = settings.env unless settings.env.nil?
+            tags[Ext::Environment::TAG_VERSION] = settings.version unless settings.version.nil?
+          end
+        end
+
+        def build_tracer_options(settings, agent_settings)
+          tracer_options = {}.tap do |opts|
+            tset = settings.tracer
+            opts[:min_spans_before_partial_flush] = tset.partial_flush.min_spans_threshold unless tset.partial_flush.min_spans_threshold.nil?
+            opts[:partial_flush] = tset.partial_flush.enabled unless tset.partial_flush.enabled.nil?
+            opts[:priority_sampling] = tset.priority_sampling unless tset.priority_sampling.nil?
+            opts[:sampler] = tset.sampler unless tset.sampler.nil?
+            opts[:writer] = tset.writer unless tset.writer.nil?
+            opts[:writer_options] = tset.writer_options if tset.writer.nil?
+          end
+
+          # Apply test mode settings if test mode is activated
+          if settings.test_mode.enabled
+            build_tracer_test_mode_options(tracer_options, settings, agent_settings)
+          else
+            tracer_options
           end
         end
 
-        def build_tracer_options(settings)
-          settings = settings.tracer
-
-          {}.tap do |opts|
-            opts[:hostname] = settings.hostname unless settings.hostname.nil?
-            opts[:min_spans_before_partial_flush] = settings.partial_flush.min_spans_threshold unless settings.partial_flush.min_spans_threshold.nil?
-            opts[:partial_flush] = settings.partial_flush.enabled unless settings.partial_flush.enabled.nil?
-            opts[:port] = settings.port unless settings.port.nil?
-            opts[:priority_sampling] = settings.priority_sampling unless settings.priority_sampling.nil?
-            opts[:sampler] = settings.sampler unless settings.sampler.nil?
-            opts[:transport_options] = settings.transport_options
-            opts[:writer] = settings.writer unless settings.writer.nil?
-            opts[:writer_options] = settings.writer_options if settings.writer.nil?
+        def build_tracer_test_mode_options(tracer_options, settings, agent_settings)
+          tracer_options.tap do |opts|
+            # Do not sample any spans for tests; all must be preserved.
+            opts[:sampler] = Datadog::AllSampler.new
+
+            # If context flush behavior is provided, use it instead.
+            opts[:context_flush] = settings.test_mode.context_flush if settings.test_mode.context_flush
+
+            # Flush traces synchronously, to guarantee they are written.
+            writer_options = settings.test_mode.writer_options || {}
+            writer_options[:agent_settings] = agent_settings if agent_settings
+            opts[:writer] = Datadog::SyncWriter.new(writer_options)
           end
         end
+
+        def build_profiler_recorder(settings)
+          event_classes = [Datadog::Profiling::Events::StackSample]
+
+          Datadog::Profiling::Recorder.new(event_classes, settings.profiling.max_events)
+        end
+
+        def build_profiler_collectors(settings, recorder)
+          [
+            Datadog::Profiling::Collectors::Stack.new(
+              recorder,
+              max_frames: settings.profiling.max_frames
+              # TODO: Provide proc that identifies Datadog worker threads?
+              # ignore_thread: settings.profiling.ignore_profiler
+            )
+          ]
+        end
+
+        def build_profiler_exporters(settings, agent_settings)
+          transport =
+            settings.profiling.exporter.transport || Datadog::Profiling::Transport::HTTP.default(
+              agent_settings: agent_settings,
+              site: settings.site,
+              api_key: settings.api_key,
+              profiling_upload_timeout_seconds: settings.profiling.upload.timeout_seconds
+            )
+
+          [Datadog::Profiling::Exporter.new(transport)]
+        end
+
+        def build_profiler_scheduler(settings, recorder, exporters)
+          Datadog::Profiling::Scheduler.new(recorder, exporters)
+        end
       end
 
       attr_reader \
         :health_metrics,
         :logger,
+        :profiler,
         :runtime_metrics,
         :tracer
 
@@ -102,8 +203,13 @@ module Datadog
         # Logger
         @logger = self.class.build_logger(settings)
 
+        agent_settings = AgentSettingsResolver.call(settings, logger: @logger)
+
         # Tracer
-        @tracer = self.class.build_tracer(settings)
+        @tracer = self.class.build_tracer(settings, agent_settings)
+
+        # Profiler
+        @profiler = self.class.build_profiler(settings, agent_settings)
 
         # Runtime metrics
         @runtime_metrics = self.class.build_runtime_metrics_worker(settings)
@@ -113,7 +219,20 @@ module Datadog
       end
 
       # Starts up components
-      def startup!(settings); end
+      def startup!(settings)
+        if settings.profiling.enabled
+          if profiler
+            @logger.debug('Profiling started')
+            profiler.start
+          else
+            # Display a warning for users who expected profiling to be enabled
+            unsupported_reason = Datadog::Profiling.unsupported_reason
+            logger.warn("Profiling was requested but is not supported, profiling disabled: #{unsupported_reason}")
+          end
+        else
+          @logger.debug('Profiling is disabled')
+        end
+      end
 
       # Shuts down all the components in use.
       # If it has another instance to compare to, it will compare
@@ -123,12 +242,20 @@ module Datadog
         # (e.g. a custom tracer instance passed in.)
         tracer.shutdown! unless replacement && tracer == replacement.tracer
 
+        # Shutdown old profiler
+        profiler.shutdown! unless profiler.nil?
+
         # Shutdown workers
-        runtime_metrics.enabled = false
-        runtime_metrics.stop(true)
+        runtime_metrics.stop(true, close_metrics: false)
 
         # Shutdown the old metrics, unless they are still being used.
         # (e.g. custom Statsd instances.)
+        #
+        # TODO: This violates the encapsulation created by Runtime::Metrics and
+        # Health::Metrics, by directly manipulating `statsd` and changing
+        # it's lifecycle management.
+        # If we need to directly have ownership of `statsd` lifecycle, we should
+        # have direct ownership of it.
         old_statsd = [
           runtime_metrics.metrics.statsd,
           health_metrics.statsd

data/lib/ddtrace/configuration/options.rb

@@ -7,17 +7,15 @@ module Datadog
     # Behavior for a configuration object that has options
     module Options
       def self.included(base)
-        base.send(:extend, ClassMethods)
-        base.send(:include, InstanceMethods)
+        base.extend(ClassMethods)
+        base.include(InstanceMethods)
       end
 
       # Class behavior for a configuration object with options
       module ClassMethods
         def options
-          @options ||= begin
-            # Allows for class inheritance of option definitions
-            superclass <= Options ? superclass.options.dup : OptionDefinitionSet.new
-          end
+          # Allows for class inheritance of option definitions
+          @options ||= superclass <= Options ? superclass.options.dup : OptionDefinitionSet.new
         end
 
         protected

data/lib/ddtrace/configuration/settings.rb

@@ -3,8 +3,10 @@ require 'ddtrace/configuration/base'
 
 require 'ddtrace/ext/analytics'
 require 'ddtrace/ext/distributed'
-require 'ddtrace/ext/runtime'
+require 'ddtrace/ext/environment'
+require 'ddtrace/ext/profiling'
 require 'ddtrace/ext/sampling'
+require 'ddtrace/ext/test'
 
 module Datadog
   module Configuration
@@ -13,9 +15,32 @@ module Datadog
     class Settings
       include Base
 
-      #
-      # Configuration options
-      #
+      def initialize(*_)
+        super
+
+        # WORKAROUND: The values for services, version, and env can get set either directly OR as a side effect of
+        # accessing tags (reading or writing). This is of course really confusing and error-prone, e.g. in an app
+        # WITHOUT this workaround where you define `DD_TAGS=env:envenvtag,service:envservicetag,version:envversiontag`
+        # and do:
+        #
+        # puts Datadog.configuration.instance_exec { "#{service} #{env} #{version}" }
+        # Datadog.configuration.tags
+        # puts Datadog.configuration.instance_exec { "#{service} #{env} #{version}" }
+        #
+        # the output will be:
+        #
+        # [empty]
+        # envservicetag envenvtag envversiontag
+        #
+        # That is -- the proper values for service/env/version are only set AFTER something accidentally or not triggers
+        # the resolution of the tags.
+        # This is really confusing, error prone, etc, so calling tags here is a really hacky but effective way to
+        # avoid this. I could not think of a better way of fixing this issue without massive refactoring of tags parsing
+        # (so that the individual service/env/version get correctly set even from their tags values, not as a side
+        # effect). Sorry :(
+        tags
+      end
+
       settings :analytics do
         option :enabled do |o|
           o.default { env_to_bool(Ext::Analytics::ENV_TRACE_ANALYTICS_ENABLED, nil) }
@@ -92,6 +117,7 @@ module Datadog
       end
 
       option :env do |o|
+        # NOTE: env also gets set as a side effect of tags. See the WORKAROUND note in #initialize for details.
         o.default { ENV.fetch(Ext::Environment::ENV_ENVIRONMENT, nil) }
         o.lazy
       end
@@ -108,6 +134,49 @@ module Datadog
         get_option(:logger).instance = logger
       end
 
+      settings :profiling do
+        option :enabled do |o|
+          o.default { env_to_bool(Ext::Profiling::ENV_ENABLED, false) }
+          o.lazy
+        end
+
+        settings :exporter do
+          option :transport
+          option :transport_options do |o|
+            o.setter do
+              # NOTE: As of April 2021 there may be a few profiler private beta customers with this setting, but since I'm
+              # marking this as deprecated before public beta, we can remove this for 1.0 without concern.
+              Datadog.logger.warn(
+                'Configuring the profiler c.profiling.exporter.transport_options is no longer needed, as the profiler ' \
+                'will reuse your existing global or tracer configuration. ' \
+                'This setting is deprecated for removal in a future ddtrace version ' \
+                '(1.0 or profiling GA, whichever comes first).'
+              )
+              nil
+            end
+            o.default { nil }
+            o.lazy
+          end
+        end
+
+        option :max_events, default: 32768
+
+        # Controls the maximum number of frames for each thread sampled. Can be tuned to avoid omitted frames in the
+        # produced profiles. Increasing this may increase the overhead of profiling.
+        option :max_frames do |o|
+          o.default { env_to_int(Ext::Profiling::ENV_MAX_FRAMES, 400) }
+          o.lazy
+        end
+
+        settings :upload do
+          option :timeout_seconds do |o|
+            o.setter { |value| value.nil? ? 30.0 : value.to_f }
+            o.default { env_to_float(Ext::Profiling::ENV_UPLOAD_TIMEOUT, 30.0) }
+            o.lazy
+          end
+        end
+      end
+
       option :report_hostname do |o|
         o.default { env_to_bool(Ext::NET::ENV_REPORT_HOSTNAME, false) }
         o.lazy
@@ -156,8 +225,17 @@ module Datadog
       end
 
       option :service do |o|
-        o.default { ENV.fetch(Ext::Environment::ENV_SERVICE, nil) }
+        # NOTE: service also gets set as a side effect of tags. See the WORKAROUND note in #initialize for details.
+        o.default { ENV.fetch(Ext::Environment::ENV_SERVICE, Ext::Environment::FALLBACK_SERVICE_NAME) }
         o.lazy
+
+        # There's a few cases where we don't want to use the fallback service name, so this helper allows us to get a
+        # nil instead so that one can do
+        # nice_service_name = Datadog.configure.service_without_fallback || nice_service_name_default
+        o.helper(:service_without_fallback) do
+          service_name = service
+          service_name unless service_name.equal?(Ext::Environment::FALLBACK_SERVICE_NAME)
+        end
       end
 
       option :site do |o|
@@ -184,16 +262,17 @@ module Datadog
 
         o.setter do |new_value, old_value|
           # Coerce keys to strings
-          string_tags = Hash[new_value.collect { |k, v| [k.to_s, v] }]
+          string_tags = new_value.collect { |k, v| [k.to_s, v] }.to_h
 
           # Cross-populate tag values with other settings
+
           self.env = string_tags[Ext::Environment::TAG_ENV] if env.nil? && string_tags.key?(Ext::Environment::TAG_ENV)
 
           if version.nil? && string_tags.key?(Ext::Environment::TAG_VERSION)
             self.version = string_tags[Ext::Environment::TAG_VERSION]
           end
 
-          if service.nil? && string_tags.key?(Ext::Environment::TAG_SERVICE)
+          if service_without_fallback.nil? && string_tags.key?(Ext::Environment::TAG_SERVICE)
             self.service = string_tags[Ext::Environment::TAG_SERVICE]
           end
 
@@ -204,6 +283,23 @@ module Datadog
         o.lazy
       end
 
+      settings :test_mode do
+        option :enabled do |o|
+          o.default { env_to_bool(Ext::Test::ENV_MODE_ENABLED, false) }
+          o.lazy
+        end
+
+        option :context_flush do |o|
+          o.default { nil }
+          o.lazy
+        end
+
+        option :writer_options do |o|
+          o.default { {} }
+          o.lazy
+        end
+      end
+
       option :time_now_provider do |o|
         o.default { ::Time.now }
 
@@ -291,6 +387,7 @@ module Datadog
       end
 
       option :version do |o|
+        # NOTE: version also gets set as a side effect of tags. See the WORKAROUND note in #initialize for details.
         o.default { ENV.fetch(Ext::Environment::ENV_VERSION, nil) }
         o.lazy
       end