ddtrace 1.1.0 → 1.12.1

data/lib/datadog/core/configuration/base.rb

@@ -1,7 +1,5 @@
-# typed: false
-
-require 'datadog/core/environment/variable_helpers'
-require 'datadog/core/configuration/options'
+require_relative '../environment/variable_helpers'
+require_relative 'options'
 
 module Datadog
   module Core
@@ -32,10 +30,13 @@ module Datadog
             option(name) do |o|
               o.default { settings_class.new }
               o.lazy
+
               o.resetter do |value|
                 value.reset! if value.respond_to?(:reset!)
                 value
               end
+
+              o.type settings_class
             end
           end
 
@@ -79,6 +80,15 @@ module Datadog
             options_hash
           end
 
+          # Retrieves a nested option from a list of symbols
+          def dig(*options)
+            raise ArgumentError, 'expected at least one option' if options.empty?
+
+            options.inject(self) do |receiver, option|
+              receiver.send(option)
+            end
+          end
+
           def reset!
             reset_options!
           end

data/lib/datadog/core/configuration/components.rb

@@ -1,23 +1,24 @@
-# typed: false
-
-require 'datadog/core/configuration/agent_settings_resolver'
-require 'datadog/core/diagnostics/environment_logger'
-require 'datadog/core/diagnostics/health'
-require 'datadog/core/logger'
-require 'datadog/core/runtime/metrics'
-require 'datadog/core/workers/runtime_metrics'
-
-require 'datadog/tracing/tracer'
-require 'datadog/tracing/flush'
-require 'datadog/tracing/sync_writer'
+require_relative 'agent_settings_resolver'
+require_relative '../diagnostics/environment_logger'
+require_relative '../diagnostics/health'
+require_relative '../logger'
+require_relative '../runtime/metrics'
+require_relative '../telemetry/client'
+require_relative '../workers/runtime_metrics'
+
+require_relative '../remote/component'
+require_relative '../../tracing/component'
+require_relative '../../profiling/component'
+require_relative '../../appsec/component'
 
 module Datadog
   module Core
     module Configuration
       # Global components for the trace library.
-      # rubocop:disable Metrics/ClassLength
       class Components
         class << self
+          include Datadog::Tracing::Component
+
           def build_health_metrics(settings)
             settings = settings.diagnostics.health_metrics
             options = { enabled: settings.enabled }
@@ -51,302 +52,43 @@ module Datadog
             Core::Workers::RuntimeMetrics.new(options)
           end
 
-          def build_tracer(settings, agent_settings)
-            # If a custom tracer has been provided, use it instead.
-            # Ignore all other options (they should already be configured.)
-            tracer = settings.tracing.instance
-            return tracer unless tracer.nil?
-
-            # Apply test mode settings if test mode is activated
-            if settings.tracing.test_mode.enabled
-              trace_flush = build_test_mode_trace_flush(settings)
-              sampler = build_test_mode_sampler
-              writer = build_test_mode_writer(settings, agent_settings)
-            else
-              trace_flush = build_trace_flush(settings)
-              sampler = build_sampler(settings)
-              writer = build_writer(settings, agent_settings)
-            end
-
-            subscribe_to_writer_events!(writer, sampler, settings.tracing.test_mode.enabled)
-
-            Tracing::Tracer.new(
-              default_service: settings.service,
-              enabled: settings.tracing.enabled,
-              trace_flush: trace_flush,
-              sampler: sampler,
-              writer: writer,
-              tags: build_tracer_tags(settings),
-            )
-          end
-
-          def build_trace_flush(settings)
-            if settings.tracing.partial_flush.enabled
-              Tracing::Flush::Partial.new(
-                min_spans_before_partial_flush: settings.tracing.partial_flush.min_spans_threshold
-              )
-            else
-              Tracing::Flush::Finished.new
-            end
-          end
-
-          # TODO: Sampler should be a top-level component.
-          # It is currently part of the Tracer initialization
-          # process, but can take a variety of options (including
-          # a fully custom instance) that makes the Tracer
-          # initialization process complex.
-          def build_sampler(settings)
-            if (sampler = settings.tracing.sampler)
-              if settings.tracing.priority_sampling == false
-                sampler
-              else
-                ensure_priority_sampling(sampler, settings)
-              end
-            elsif settings.tracing.priority_sampling == false
-              Tracing::Sampling::RuleSampler.new(
-                rate_limit: settings.tracing.sampling.rate_limit,
-                default_sample_rate: settings.tracing.sampling.default_rate
-              )
-            else
-              Tracing::Sampling::PrioritySampler.new(
-                base_sampler: Tracing::Sampling::AllSampler.new,
-                post_sampler: Tracing::Sampling::RuleSampler.new(
-                  rate_limit: settings.tracing.sampling.rate_limit,
-                  default_sample_rate: settings.tracing.sampling.default_rate
-                )
-              )
-            end
-          end
-
-          def ensure_priority_sampling(sampler, settings)
-            if sampler.is_a?(Tracing::Sampling::PrioritySampler)
-              sampler
-            else
-              Tracing::Sampling::PrioritySampler.new(
-                base_sampler: sampler,
-                post_sampler: Tracing::Sampling::RuleSampler.new(
-                  rate_limit: settings.tracing.sampling.rate_limit,
-                  default_sample_rate: settings.tracing.sampling.default_rate
-                )
-              )
-            end
-          end
-
-          # TODO: Writer should be a top-level component.
-          # It is currently part of the Tracer initialization
-          # process, but can take a variety of options (including
-          # a fully custom instance) that makes the Tracer
-          # initialization process complex.
-          def build_writer(settings, agent_settings)
-            if (writer = settings.tracing.writer)
-              return writer
-            end
-
-            Tracing::Writer.new(agent_settings: agent_settings, **settings.tracing.writer_options)
-          end
-
-          def subscribe_to_writer_events!(writer, sampler, test_mode)
-            return unless writer.respond_to?(:events) # Check if it's a custom, external writer
-
-            writer.events.after_send.subscribe(&WRITER_RECORD_ENVIRONMENT_INFORMATION_CALLBACK)
-
-            return unless sampler.is_a?(Tracing::Sampling::PrioritySampler)
-
-            # DEV: We need to ignore priority sampling updates coming from the agent in test mode
-            # because test mode wants to *unconditionally* sample all traces.
-            #
-            # This can cause trace metrics to be overestimated, but that's a trade-off we take
-            # here to achieve 100% sampling rate.
-            return if test_mode
-
-            writer.events.after_send.subscribe(&writer_update_priority_sampler_rates_callback(sampler))
-          end
-
-          WRITER_RECORD_ENVIRONMENT_INFORMATION_CALLBACK = lambda do |_, responses|
-            Core::Diagnostics::EnvironmentLogger.log!(responses)
-          end
-
-          # Create new lambda for writer callback,
-          # capture the current sampler in the callback closure.
-          def writer_update_priority_sampler_rates_callback(sampler)
-            lambda do |_, responses|
-              response = responses.last
-
-              next unless response && !response.internal_error? && response.service_rates
-
-              sampler.update(response.service_rates)
-            end
-          end
-
-          def build_profiler(settings, agent_settings, tracer)
-            return unless settings.profiling.enabled
-
-            # Workaround for weird dependency direction: the Core::Configuration::Components class currently has a
-            # dependency on individual products, in this case the Profiler.
-            # (Note "currently": in the future we want to change this so core classes don't depend on specific products)
-            #
-            # If the current file included a `require 'datadog/profiler'` at its beginning, we would generate circular
-            # requires when used from profiling:
-            #
-            # datadog/profiling
-            #     └─requires─> datadog/core
-            #                      └─requires─> datadog/core/configuration/components
-            #                                       └─requires─> datadog/profiling       # Loop!
-            #
-            # ...thus in #1998 we removed such a require.
-            #
-            # On the other hand, if datadog/core is loaded by a different product and no general `require 'ddtrace'` is
-            # done, then profiling may not be loaded, and thus to avoid this issue we do a require here (which is a
-            # no-op if profiling is already loaded).
-            require 'datadog/profiling'
-            return unless Profiling.supported?
-
-            unless defined?(Profiling::Tasks::Setup)
-              # In #1545 a user reported a NameError due to this constant being uninitialized
-              # I've documented my suspicion on why that happened in
-              # https://github.com/DataDog/dd-trace-rb/issues/1545#issuecomment-856049025
-              #
-              # > Thanks for the info! It seems to feed into my theory: there's two moments in the code where we check if
-              # > profiler is "supported": 1) when loading ddtrace (inside preload) and 2) when starting the profile
-              # > after Datadog.configure gets run.
-              # > The problem is that the code assumes that both checks 1) and 2) will always reach the same conclusion:
-              # > either profiler is supported, or profiler is not supported.
-              # > In the problematic case, it looks like in your case check 1 decides that profiler is not
-              # > supported => doesn't load it, and then check 2 decides that it is => assumes it is loaded and tries to
-              # > start it.
-              #
-              # I was never able to validate if this was the issue or why exactly .supported? would change its mind BUT
-              # just in case it happens again, I've left this check which avoids breaking the user's application AND
-              # would instead direct them to report it to us instead, so that we can investigate what's wrong.
-              #
-              # TODO: As of June 2021, most checks in .supported? are related to the google-protobuf gem; so it's
-              # very likely that it was the origin of the issue we saw. Thus, if, as planned we end up moving away from
-              # protobuf OR enough time has passed and no users saw the issue again, we can remove this check altogether.
-              Datadog.logger.error(
-                'Profiling was marked as supported and enabled, but setup task was not loaded properly. ' \
-                'Please report this at https://github.com/DataDog/dd-trace-rb/blob/master/CONTRIBUTING.md#found-a-bug'
-              )
-
-              return
-            end
-
-            # Load extensions needed to support some of the Profiling features
-            Profiling::Tasks::Setup.new.run
-
-            # NOTE: Please update the Initialization section of ProfilingDevelopment.md with any changes to this method
-
-            trace_identifiers_helper = Profiling::TraceIdentifiers::Helper.new(
-              tracer: tracer,
-              endpoint_collection_enabled: settings.profiling.advanced.endpoint.collection.enabled
-            )
-
-            # TODO: It's a bit weird to treat this collector differently from others. See the TODO on the
-            # Datadog::Profiling::Recorder class for a discussion of this choice.
-            if settings.profiling.advanced.code_provenance_enabled
-              code_provenance_collector =
-                Profiling::Collectors::CodeProvenance.new
-            end
-
-            recorder = build_profiler_recorder(settings, code_provenance_collector)
-            collectors = build_profiler_collectors(settings, recorder, trace_identifiers_helper)
-            exporters = build_profiler_exporters(settings, agent_settings)
-            scheduler = build_profiler_scheduler(settings, recorder, exporters)
-
-            Profiling::Profiler.new(collectors, scheduler)
-          end
-
-          private
-
-          def build_tracer_tags(settings)
-            settings.tags.dup.tap do |tags|
-              tags[Core::Environment::Ext::TAG_ENV] = settings.env unless settings.env.nil?
-              tags[Core::Environment::Ext::TAG_VERSION] = settings.version unless settings.version.nil?
+          def build_telemetry(settings, agent_settings, logger)
+            enabled = settings.telemetry.enabled
+            if agent_settings.adapter != Datadog::Transport::Ext::HTTP::ADAPTER
+              enabled = false
+              logger.debug { "Telemetry disabled. Agent network adapter not supported: #{agent_settings.adapter}" }
             end
-          end
 
-          def build_test_mode_trace_flush(settings)
-            # If context flush behavior is provided, use it instead.
-            settings.tracing.test_mode.trace_flush || build_trace_flush(settings)
-          end
-
-          def build_test_mode_sampler
-            # Do not sample any spans for tests; all must be preserved.
-            # Set priority sampler to ensure the agent doesn't drop any traces.
-            Tracing::Sampling::PrioritySampler.new(
-              base_sampler: Tracing::Sampling::AllSampler.new,
-              post_sampler: Tracing::Sampling::AllSampler.new
-            )
-          end
-
-          def build_test_mode_writer(settings, agent_settings)
-            # Flush traces synchronously, to guarantee they are written.
-            writer_options = settings.tracing.test_mode.writer_options || {}
-            Tracing::SyncWriter.new(agent_settings: agent_settings, **writer_options)
-          end
-
-          def build_profiler_recorder(settings, code_provenance_collector)
-            event_classes = [Profiling::Events::StackSample]
-
-            Profiling::Recorder.new(
-              event_classes,
-              settings.profiling.advanced.max_events,
-              code_provenance_collector: code_provenance_collector
-            )
-          end
-
-          def build_profiler_collectors(settings, recorder, trace_identifiers_helper)
-            [
-              Profiling::Collectors::OldStack.new(
-                recorder,
-                trace_identifiers_helper: trace_identifiers_helper,
-                max_frames: settings.profiling.advanced.max_frames
-                # TODO: Provide proc that identifies Datadog worker threads?
-                # ignore_thread: settings.profiling.ignore_profiler
-              )
-            ]
-          end
-
-          def build_profiler_exporters(settings, agent_settings)
-            transport =
-              settings.profiling.exporter.transport || Profiling::Transport::HTTP.default(
-                agent_settings: agent_settings,
-                site: settings.site,
-                api_key: settings.api_key,
-                profiling_upload_timeout_seconds: settings.profiling.upload.timeout_seconds
-              )
-
-            [Profiling::Exporter.new(transport)]
-          end
-
-          def build_profiler_scheduler(settings, recorder, exporters)
-            Profiling::Scheduler.new(recorder, exporters)
+            Telemetry::Client.new(enabled: enabled)
           end
         end
 
         attr_reader \
           :health_metrics,
           :logger,
+          :remote,
           :profiler,
           :runtime_metrics,
-          :tracer
+          :telemetry,
+          :tracer,
+          :appsec
 
         def initialize(settings)
-          # Logger
           @logger = self.class.build_logger(settings)
 
           agent_settings = AgentSettingsResolver.call(settings, logger: @logger)
 
-          # Tracer
+          @remote = Remote::Component.build(settings, agent_settings)
           @tracer = self.class.build_tracer(settings, agent_settings)
-
-          # Profiler
-          @profiler = self.class.build_profiler(settings, agent_settings, @tracer)
-
-          # Runtime metrics
+          @profiler = Datadog::Profiling::Component.build_profiler_component(
+            settings: settings,
+            agent_settings: agent_settings,
+            optional_tracer: @tracer,
+          )
           @runtime_metrics = self.class.build_runtime_metrics_worker(settings)
-
-          # Health metrics
           @health_metrics = self.class.build_health_metrics(settings)
+          @telemetry = self.class.build_telemetry(settings, agent_settings, logger)
+          @appsec = Datadog::AppSec::Component.build_appsec_component(settings)
         end
 
         # Starts up components
@@ -369,6 +111,12 @@ module Datadog
         # If it has another instance to compare to, it will compare
         # and avoid tearing down parts still in use.
         def shutdown!(replacement = nil)
+          # Shutdown remote configuration
+          remote.shutdown! if remote
+
+          # Decommission AppSec
+          appsec.shutdown! if appsec
+
           # Shutdown the old tracer, unless it's still being used.
           # (e.g. a custom tracer instance passed in.)
           tracer.shutdown! unless replacement && tracer == replacement.tracer
@@ -403,9 +151,11 @@ module Datadog
 
           unused_statsd = (old_statsd - (old_statsd & new_statsd))
           unused_statsd.each(&:close)
+
+          telemetry.stop!
+          telemetry.emit_closing! unless replacement
         end
       end
-      # rubocop:enable Metrics/ClassLength
     end
   end
 end

data/lib/datadog/core/configuration/dependency_resolver.rb

@@ -1,4 +1,4 @@
-# typed: false
+# frozen_string_literal: true
 
 require 'tsort'
 

data/lib/datadog/core/configuration/ext.rb

@@ -0,0 +1,24 @@
+module Datadog
+  module Core
+    module Configuration
+      # Constants for configuration settings
+      # e.g. Env vars, default values, enums, etc...
+      module Ext
+        # @public_api
+        module Diagnostics
+          ENV_DEBUG_ENABLED = 'DD_TRACE_DEBUG'.freeze
+          ENV_HEALTH_METRICS_ENABLED = 'DD_HEALTH_METRICS_ENABLED'.freeze
+          ENV_STARTUP_LOGS_ENABLED = 'DD_TRACE_STARTUP_LOGS'.freeze
+        end
+
+        module Metrics
+          ENV_DEFAULT_PORT = 'DD_METRIC_AGENT_PORT'.freeze
+        end
+
+        module Transport
+          ENV_DEFAULT_HOST = 'DD_AGENT_HOST'.freeze
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/configuration/option.rb

@@ -1,4 +1,4 @@
-# typed: true
+# frozen_string_literal: true
 
 module Datadog
   module Core

data/lib/datadog/core/configuration/option_definition.rb

@@ -1,6 +1,6 @@
-# typed: true
+# frozen_string_literal: true
 
-require 'datadog/core/configuration/option'
+require_relative 'option'
 
 module Datadog
   module Core
@@ -17,7 +17,8 @@ module Datadog
           :name,
           :on_set,
           :resetter,
-          :setter
+          :setter,
+          :type
 
         def initialize(name, meta = {}, &block)
           @default = meta[:default]
@@ -28,6 +29,7 @@ module Datadog
           @on_set = meta[:on_set]
           @resetter = meta[:resetter]
           @setter = meta[:setter] || block || IDENTITY
+          @type = meta[:type]
         end
 
         # Creates a new Option, bound to the context provided.
@@ -51,6 +53,7 @@ module Datadog
             @on_set = nil
             @resetter = nil
             @setter = OptionDefinition::IDENTITY
+            @type = nil
 
             # If options were supplied, apply them.
             apply_options!(options)
@@ -91,6 +94,10 @@ module Datadog
             @setter = block
           end
 
+          def type(value = nil)
+            @type = value
+          end
+
           # For applying options for OptionDefinition
           def apply_options!(options = {})
             return if options.nil? || options.empty?
@@ -102,6 +109,7 @@ module Datadog
             on_set(&options[:on_set]) if options.key?(:on_set)
             resetter(&options[:resetter]) if options.key?(:resetter)
             setter(&options[:setter]) if options.key?(:setter)
+            type(&options[:type]) if options.key?(:type)
           end
 
           def to_definition
@@ -116,7 +124,8 @@ module Datadog
               lazy: @lazy,
               on_set: @on_set,
               resetter: @resetter,
-              setter: @setter
+              setter: @setter,
+              type: @type
             }
           end
         end

data/lib/datadog/core/configuration/option_definition_set.rb

@@ -1,6 +1,6 @@
-# typed: true
+# frozen_string_literal: true
 
-require 'datadog/core/configuration/dependency_resolver'
+require_relative 'dependency_resolver'
 
 module Datadog
   module Core

data/lib/datadog/core/configuration/option_set.rb

@@ -1,4 +1,4 @@
-# typed: true
+# frozen_string_literal: true
 
 module Datadog
   module Core

data/lib/datadog/core/configuration/options.rb

@@ -1,8 +1,6 @@
-# typed: false
-
-require 'datadog/core/configuration/option_set'
-require 'datadog/core/configuration/option_definition'
-require 'datadog/core/configuration/option_definition_set'
+require_relative 'option_set'
+require_relative 'option_definition'
+require_relative 'option_definition_set'
 
 module Datadog
   module Core