datadog 2.0.0.beta1

data/lib/datadog/core/configuration/components.rb

@@ -0,0 +1,177 @@
+# frozen_string_literal: true
+
+require_relative 'agent_settings_resolver'
+require_relative 'ext'
+require_relative '../diagnostics/environment_logger'
+require_relative '../diagnostics/health'
+require_relative '../logger'
+require_relative '../runtime/metrics'
+require_relative '../telemetry/client'
+require_relative '../workers/runtime_metrics'
+
+require_relative '../remote/component'
+require_relative '../../tracing/component'
+require_relative '../../profiling/component'
+require_relative '../../appsec/component'
+
+module Datadog
+  module Core
+    module Configuration
+      # Global components for the trace library.
+      class Components
+        class << self
+          include Datadog::Tracing::Component
+
+          def build_health_metrics(settings)
+            settings = settings.health_metrics
+            options = { enabled: settings.enabled }
+            options[:statsd] = settings.statsd unless settings.statsd.nil?
+
+            Core::Diagnostics::Health::Metrics.new(**options)
+          end
+
+          def build_logger(settings)
+            logger = settings.logger.instance || Core::Logger.new($stdout)
+            logger.level = settings.diagnostics.debug ? ::Logger::DEBUG : settings.logger.level
+
+            logger
+          end
+
+          def build_runtime_metrics(settings)
+            options = { enabled: settings.runtime_metrics.enabled }
+            options[:statsd] = settings.runtime_metrics.statsd unless settings.runtime_metrics.statsd.nil?
+            options[:services] = [settings.service] unless settings.service.nil?
+
+            Core::Runtime::Metrics.new(**options)
+          end
+
+          def build_runtime_metrics_worker(settings)
+            # NOTE: Should we just ignore building the worker if its not enabled?
+            options = settings.runtime_metrics.opts.merge(
+              enabled: settings.runtime_metrics.enabled,
+              metrics: build_runtime_metrics(settings)
+            )
+
+            Core::Workers::RuntimeMetrics.new(options)
+          end
+
+          def build_telemetry(settings, agent_settings, logger)
+            enabled = settings.telemetry.enabled
+            if agent_settings.adapter != Datadog::Core::Configuration::Ext::Agent::HTTP::ADAPTER
+              enabled = false
+              logger.debug { "Telemetry disabled. Agent network adapter not supported: #{agent_settings.adapter}" }
+            end
+
+            Telemetry::Client.new(
+              enabled: enabled,
+              heartbeat_interval_seconds: settings.telemetry.heartbeat_interval_seconds
+            )
+          end
+        end
+
+        include Datadog::Tracing::Component::InstanceMethods
+
+        attr_reader \
+          :health_metrics,
+          :logger,
+          :remote,
+          :profiler,
+          :runtime_metrics,
+          :telemetry,
+          :tracer,
+          :appsec
+
+        def initialize(settings)
+          @logger = self.class.build_logger(settings)
+          @environment_logger_extra = {}
+
+          # This agent_settings is intended for use within Core. If you require
+          # agent_settings within a product outside of core you should extend
+          # the Core resolver from within your product/component's namespace.
+          agent_settings = AgentSettingsResolver.call(settings, logger: @logger)
+
+          @remote = Remote::Component.build(settings, agent_settings)
+          @tracer = self.class.build_tracer(settings, agent_settings, logger: @logger)
+
+          @profiler, profiler_logger_extra = Datadog::Profiling::Component.build_profiler_component(
+            settings: settings,
+            agent_settings: agent_settings,
+            optional_tracer: @tracer,
+          )
+          @environment_logger_extra.merge!(profiler_logger_extra) if profiler_logger_extra
+
+          @runtime_metrics = self.class.build_runtime_metrics_worker(settings)
+          @health_metrics = self.class.build_health_metrics(settings)
+          @telemetry = self.class.build_telemetry(settings, agent_settings, logger)
+          @appsec = Datadog::AppSec::Component.build_appsec_component(settings)
+
+          self.class.configure_tracing(settings)
+        end
+
+        # Starts up components
+        def startup!(settings)
+          if settings.profiling.enabled
+            if profiler
+              profiler.start
+            else
+              # Display a warning for users who expected profiling to be enabled
+              unsupported_reason = Profiling.unsupported_reason
+              logger.warn("Profiling was requested but is not supported, profiling disabled: #{unsupported_reason}")
+            end
+          end
+
+          Core::Diagnostics::EnvironmentLogger.collect_and_log!(@environment_logger_extra)
+        end
+
+        # Shuts down all the components in use.
+        # If it has another instance to compare to, it will compare
+        # and avoid tearing down parts still in use.
+        def shutdown!(replacement = nil)
+          # Shutdown remote configuration
+          remote.shutdown! if remote
+
+          # Decommission AppSec
+          appsec.shutdown! if appsec
+
+          # Shutdown the old tracer, unless it's still being used.
+          # (e.g. a custom tracer instance passed in.)
+          tracer.shutdown! unless replacement && tracer == replacement.tracer
+
+          # Shutdown old profiler
+          profiler.shutdown! unless profiler.nil?
+
+          # Shutdown workers
+          runtime_metrics.stop(true, close_metrics: false)
+
+          # Shutdown the old metrics, unless they are still being used.
+          # (e.g. custom Statsd instances.)
+          #
+          # TODO: This violates the encapsulation created by Runtime::Metrics and
+          # Health::Metrics, by directly manipulating `statsd` and changing
+          # it's lifecycle management.
+          # If we need to directly have ownership of `statsd` lifecycle, we should
+          # have direct ownership of it.
+          old_statsd = [
+            runtime_metrics.metrics.statsd,
+            health_metrics.statsd
+          ].compact.uniq
+
+          new_statsd =  if replacement
+                          [
+                            replacement.runtime_metrics.metrics.statsd,
+                            replacement.health_metrics.statsd
+                          ].compact.uniq
+                        else
+                          []
+                        end
+
+          unused_statsd = (old_statsd - (old_statsd & new_statsd))
+          unused_statsd.each(&:close)
+
+          telemetry.stop!
+          telemetry.emit_closing! unless replacement
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/configuration/ext.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Configuration
+      # Constants for configuration settings
+      # e.g. Env vars, default values, enums, etc...
+      module Ext
+        # @public_api
+        module Diagnostics
+          ENV_DEBUG_ENABLED = 'DD_TRACE_DEBUG'
+          ENV_HEALTH_METRICS_ENABLED = 'DD_HEALTH_METRICS_ENABLED'
+          ENV_STARTUP_LOGS_ENABLED = 'DD_TRACE_STARTUP_LOGS'
+        end
+
+        module Metrics
+          ENV_DEFAULT_PORT = 'DD_METRIC_AGENT_PORT'
+        end
+
+        module Agent
+          ENV_DEFAULT_HOST = 'DD_AGENT_HOST'
+          # Some env vars have "trace" in them, but they apply to all products
+          ENV_DEFAULT_PORT = 'DD_TRACE_AGENT_PORT'
+          ENV_DEFAULT_TIMEOUT_SECONDS = 'DD_TRACE_AGENT_TIMEOUT_SECONDS'
+          ENV_DEFAULT_URL = 'DD_TRACE_AGENT_URL'
+
+          module HTTP
+            ADAPTER = :net_http
+            DEFAULT_HOST = '127.0.0.1'
+            DEFAULT_PORT = 8126
+            DEFAULT_USE_SSL = false
+            DEFAULT_TIMEOUT_SECONDS = 30
+          end
+
+          # @public_api
+          module UnixSocket
+            ADAPTER = :unix
+            DEFAULT_PATH = '/var/run/datadog/apm.socket'
+            DEFAULT_TIMEOUT_SECONDS = 1
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/configuration/option.rb

@@ -0,0 +1,319 @@
+# frozen_string_literal: true
+
+require_relative '../utils/safe_dup'
+
+module Datadog
+  module Core
+    module Configuration
+      # Represents an instance of an integration configuration option
+      # @public_api
+      class Option
+        attr_reader :definition
+
+        # Option setting precedence.
+        module Precedence
+          # Represents an Option precedence level.
+          # Each precedence has a `numeric` value; higher values means higher precedence.
+          # `name` is for inspection purposes only.
+          Value = Struct.new(:numeric, :name) do
+            include Comparable
+
+            def <=>(other)
+              return nil unless other.is_a?(Value)
+
+              numeric <=> other.numeric
+            end
+          end
+
+          # Remote configuration provided through the Datadog app.
+          REMOTE_CONFIGURATION = Value.new(2, :remote_configuration).freeze
+
+          # Configuration provided in Ruby code, in this same process
+          # or via Environment variable
+          PROGRAMMATIC = Value.new(1, :programmatic).freeze
+
+          # Configuration that comes from default values
+          DEFAULT = Value.new(0, :default).freeze
+
+          # All precedences, sorted from highest to lowest
+          LIST = [REMOTE_CONFIGURATION, PROGRAMMATIC, DEFAULT].sort.reverse.freeze
+        end
+
+        def initialize(definition, context)
+          @definition = definition
+          @context = context
+          @value = nil
+          @is_set = false
+
+          # One value is stored per precedence, to allow unsetting a higher
+          # precedence value and falling back to a lower precedence one.
+          @value_per_precedence = Hash.new(UNSET)
+
+          # Lowest precedence, to allow for `#set` to always succeed for a brand new `Option` instance.
+          @precedence_set = Precedence::DEFAULT
+        end
+
+        # Overrides the current value for this option if the `precedence` is equal or higher than
+        # the previously set value.
+        # The first call to `#set` will always store the value regardless of precedence.
+        #
+        # @param value [Object] the new value to be associated with this option
+        # @param precedence [Precedence] from what precedence order this new value comes from
+        def set(value, precedence: Precedence::PROGRAMMATIC)
+          # Is there a higher precedence value set?
+          if @precedence_set > precedence
+            # This should be uncommon, as higher precedence values tend to
+            # happen later in the application lifecycle.
+            Datadog.logger.info do
+              "Option '#{definition.name}' not changed to '#{value}' (precedence: #{precedence.name}) because the higher " \
+                "precedence value '#{@value}' (precedence: #{@precedence_set.name}) was already set."
+            end
+
+            # But if it happens, we have to store the lower precedence value `value`
+            # because it's possible to revert to it by `#unset`ting
+            # the existing, higher-precedence value.
+            # Effectively, we always store one value pre precedence.
+            @value_per_precedence[precedence] = value
+
+            return @value
+          end
+
+          internal_set(value, precedence)
+        end
+
+        def unset(precedence)
+          @value_per_precedence[precedence] = UNSET
+
+          # If we are unsetting the currently active value, we have to restore
+          # a lower precedence one...
+          if precedence == @precedence_set
+            # Find a lower precedence value that is already set.
+            Precedence::LIST.each do |p|
+              # DEV: This search can be optimized, but the list is small, and unset is
+              # DEV: only called from direct user interaction in the Datadog UI.
+              next unless p < precedence
+
+              # Look for value that is set.
+              # The hash `@value_per_precedence` has a custom default value of `UNSET`.
+              if (value = @value_per_precedence[p]) != UNSET
+                internal_set(value, p)
+                return nil
+              end
+            end
+
+            # If no value is left to fall back on, reset this option
+            reset
+          end
+
+          # ... otherwise, we are either unsetting a higher precedence value that is not
+          # yet set, thus there's nothing to do; or we are unsetting a lower precedence
+          # value, which also does not change the current value.
+        end
+
+        def get
+          if @is_set
+            @value
+          else
+            set_value_from_env_or_default
+          end
+        end
+
+        def reset
+          @value = if definition.resetter
+                     # Don't change @is_set to false; custom resetters are
+                     # responsible for changing @value back to a good state.
+                     # Setting @is_set = false would cause a default to be applied.
+                     context_exec(@value, &definition.resetter)
+                   else
+                     @is_set = false
+                     nil
+                   end
+
+          # Reset back to the lowest precedence, to allow all `set`s to succeed right after a reset.
+          @precedence_set = Precedence::DEFAULT
+          # Reset all stored values
+          @value_per_precedence = Hash.new(UNSET)
+        end
+
+        def default_value
+          if definition.default.instance_of?(Proc)
+            context_eval(&definition.default)
+          else
+            definition.default_proc || Core::Utils::SafeDup.frozen_or_dup(definition.default)
+          end
+        end
+
+        def default_precedence?
+          precedence_set == Precedence::DEFAULT
+        end
+
+        private
+
+        def coerce_env_variable(value)
+          return context_exec(value, &@definition.env_parser) if @definition.env_parser
+
+          case @definition.type
+          when :hash
+            values = value.split(',') # By default we only want to support comma separated strings
+
+            values.map! do |v|
+              v.gsub!(/\A[\s,]*|[\s,]*\Z/, '')
+
+              v.empty? ? nil : v
+            end
+
+            values.compact!
+            values.each.with_object({}) do |v, hash|
+              pair = v.split(':', 2)
+              hash[pair[0]] = pair[1]
+            end
+          when :int
+            Integer(value, 10)
+          when :float
+            Float(value)
+          when :array
+            values = value.split(',')
+
+            values.map! do |v|
+              v.gsub!(/\A[\s,]*|[\s,]*\Z/, '')
+
+              v.empty? ? nil : v
+            end
+
+            values.compact!
+            values
+          when :bool
+            string_value = value.strip
+            string_value = string_value.downcase
+            string_value == 'true' || string_value == '1' # rubocop:disable Style/MultipleComparison
+          when :string, NilClass
+            value
+          else
+            raise InvalidDefinitionError,
+              "The option #{@definition.name} is using an unsupported type option for env coercion `#{@definition.type}`"
+          end
+        end
+
+        # For errors caused by illegal option declarations
+        class InvalidDefinitionError < StandardError
+        end
+
+        def validate_type(value)
+          raise_error = false
+
+          valid_type = validate(@definition.type, value)
+
+          unless valid_type
+            raise_error = if @definition.type_options[:nilable]
+                            !value.is_a?(NilClass)
+                          else
+                            true
+                          end
+          end
+
+          if raise_error
+            error_msg = if @definition.type_options[:nilable]
+                          "The setting `#{@definition.name}` inside your app's `Datadog.configure` block expects a "\
+                          "#{@definition.type} or `nil`, but a `#{value.class}` was provided (#{value.inspect})."\
+                        else
+                          "The setting `#{@definition.name}` inside your app's `Datadog.configure` block expects a "\
+                          "#{@definition.type}, but a `#{value.class}` was provided (#{value.inspect})."\
+                        end
+
+            error_msg = "#{error_msg} Please update your `configure` block. "
+
+            raise ArgumentError, error_msg
+          end
+
+          value
+        end
+
+        def validate(type, value)
+          case type
+          when :string
+            value.is_a?(String)
+          when :int
+            value.is_a?(Integer)
+          when :float
+            value.is_a?(Float) || value.is_a?(Integer) || value.is_a?(Rational)
+          when :array
+            value.is_a?(Array)
+          when :hash
+            value.is_a?(Hash)
+          when :bool
+            value.is_a?(TrueClass) || value.is_a?(FalseClass)
+          when :proc
+            value.is_a?(Proc)
+          when :symbol
+            value.is_a?(Symbol)
+          when NilClass
+            true # No validation is performed when option is typeless
+          else
+            raise ArgumentError, "The option #{@definition.name} is using an unsupported type option `#{@definition.type}`"
+          end
+        end
+
+        # Directly manipulates the current value and currently set precedence.
+        def internal_set(value, precedence)
+          old_value = @value
+          (@value = context_exec(validate_type(value), old_value, &definition.setter)).tap do |v|
+            @is_set = true
+            @precedence_set = precedence
+            # Store original value to ensure we can always safely call `#internal_set`
+            # when restoring a value from `@value_per_precedence`, and we are only running `definition.setter`
+            # on the original value, not on a valud that has already been processed by `definition.setter`.
+            @value_per_precedence[precedence] = value
+            context_exec(v, old_value, precedence, &definition.after_set) if definition.after_set
+          end
+        end
+
+        def context_exec(*args, &block)
+          @context.instance_exec(*args, &block)
+        end
+
+        def context_eval(&block)
+          @context.instance_eval(&block)
+        end
+
+        def set_value_from_env_or_default
+          value = nil
+          precedence = nil
+          effective_env = nil
+
+          if definition.env && ENV[definition.env]
+            effective_env = definition.env
+            value = coerce_env_variable(ENV[definition.env])
+            precedence = Precedence::PROGRAMMATIC
+          end
+
+          if value.nil? && definition.deprecated_env && ENV[definition.deprecated_env]
+            effective_env = definition.deprecated_env
+            value = coerce_env_variable(ENV[definition.deprecated_env])
+            precedence = Precedence::PROGRAMMATIC
+
+            Datadog::Core.log_deprecation do
+              "#{definition.deprecated_env} environment variable is deprecated, use #{definition.env} instead."
+            end
+          end
+
+          option_value = value.nil? ? default_value : value
+
+          set(option_value, precedence: precedence || Precedence::DEFAULT)
+        rescue ArgumentError
+          raise ArgumentError,
+            "Expected environment variable #{effective_env} to be a #{@definition.type}, " \
+                              "but '#{ENV[effective_env]}' was provided"
+        end
+
+        # Used for testing
+        attr_reader :precedence_set
+        private :precedence_set
+
+        # Anchor object that represents a value that is not set.
+        # This is necessary because `nil` is a valid value to be set.
+        UNSET = Object.new
+        private_constant :UNSET
+      end
+    end
+  end
+end

data/lib/datadog/core/configuration/option_definition.rb

@@ -0,0 +1,165 @@
+# frozen_string_literal: true
+
+require_relative 'option'
+
+module Datadog
+  module Core
+    module Configuration
+      # Represents a definition for an integration configuration option
+      class OptionDefinition
+        IDENTITY = ->(new_value, _old_value) { new_value }
+
+        attr_reader \
+          :default,
+          :default_proc,
+          :env,
+          :deprecated_env,
+          :env_parser,
+          :name,
+          :after_set,
+          :resetter,
+          :setter,
+          :type,
+          :type_options
+
+        def initialize(name, meta = {}, &block)
+          @default = meta[:default]
+          @default_proc = meta[:default_proc]
+          @env = meta[:env]
+          @deprecated_env = meta[:deprecated_env]
+          @env_parser = meta[:env_parser]
+          @name = name.to_sym
+          @after_set = meta[:after_set]
+          @resetter = meta[:resetter]
+          @setter = meta[:setter] || block || IDENTITY
+          @type = meta[:type]
+          @type_options = meta[:type_options]
+        end
+
+        # Creates a new Option, bound to the context provided.
+        def build(context)
+          Option.new(self, context)
+        end
+
+        # Acts as DSL for building OptionDefinitions
+        # @public_api
+        class Builder
+          class InvalidOptionError < StandardError; end
+
+          attr_reader \
+            :helpers
+
+          def initialize(name, options = {})
+            @env = nil
+            @deprecated_env = nil
+            @env_parser = nil
+            @default = nil
+            @default_proc = nil
+            @helpers = {}
+            @name = name.to_sym
+            @after_set = nil
+            @resetter = nil
+            @setter = OptionDefinition::IDENTITY
+            @type = nil
+            @type_options = {}
+            # If options were supplied, apply them.
+            apply_options!(options)
+
+            # Apply block if given.
+            yield(self) if block_given?
+
+            validate_options!
+          end
+
+          def env(value)
+            @env = value
+          end
+
+          def deprecated_env(value)
+            @deprecated_env = value
+          end
+
+          def env_parser(&block)
+            @env_parser = block
+          end
+
+          def default(value = nil, &block)
+            @default = block || value
+          end
+
+          def default_proc(&block)
+            @default_proc = block
+          end
+
+          def helper(name, *_args, &block)
+            @helpers[name] = block
+          end
+
+          def after_set(&block)
+            @after_set = block
+          end
+
+          def resetter(&block)
+            @resetter = block
+          end
+
+          def setter(&block)
+            @setter = block
+          end
+
+          def type(value, nilable: false)
+            @type = value
+            @type_options = { nilable: nilable }
+
+            value
+          end
+
+          # For applying options for OptionDefinition
+          def apply_options!(options = {})
+            return if options.nil? || options.empty?
+
+            default(options[:default]) if options.key?(:default)
+            default_proc(&options[:default_proc]) if options.key?(:default_proc)
+            env(options[:env]) if options.key?(:env)
+            deprecated_env(options[:deprecated_env]) if options.key?(:deprecated_env)
+            env_parser(&options[:env_parser]) if options.key?(:env_parser)
+            after_set(&options[:after_set]) if options.key?(:after_set)
+            resetter(&options[:resetter]) if options.key?(:resetter)
+            setter(&options[:setter]) if options.key?(:setter)
+            type(options[:type], **(options[:type_options] || {})) if options.key?(:type)
+          end
+
+          def to_definition
+            OptionDefinition.new(@name, meta)
+          end
+
+          def meta
+            {
+              default: @default,
+              default_proc: @default_proc,
+              env: @env,
+              deprecated_env: @deprecated_env,
+              env_parser: @env_parser,
+              after_set: @after_set,
+              resetter: @resetter,
+              setter: @setter,
+              type: @type,
+              type_options: @type_options
+            }
+          end
+
+          private
+
+          def validate_options!
+            if !@default.nil? && @default_proc
+              raise InvalidOptionError,
+                'Using `default` and `default_proc` is not allowed. Please use one or the other.' \
+                                'If you want to store a block as the default value use `default_proc`'\
+                                ' otherwise use `default`'
+            end
+          end
+        end
+      end
+    end
+  end
+end