datadog 2.0.0.beta1

data/lib/datadog/core/buffer/random.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Buffer
+      # Buffer that accumulates items for a consumer.
+      # Consumption can happen from a different thread.
+
+      # Buffer that stores objects. The buffer has a maximum size and when
+      # the buffer is full, a random object is discarded.
+      class Random
+        def initialize(max_size)
+          @max_size = max_size
+          @items = []
+          @closed = false
+        end
+
+        # Add a new ``item`` in the local queue. This method doesn't block the execution
+        # even if the buffer is full.
+        #
+        # When the buffer is full, we try to ensure that we are fairly choosing newly
+        # pushed items by randomly inserting them into the buffer slots. This discards
+        # old items randomly while trying to ensure that recent items are still captured.
+        def push(item)
+          return if closed?
+
+          full? ? replace!(item) : add!(item)
+          item
+        end
+
+        # A bulk push alternative to +#push+. Use this method if
+        # pushing more than one item for efficiency.
+        def concat(items)
+          return if closed?
+
+          # Segment items into underflow and overflow
+          underflow, overflow = overflow_segments(items)
+
+          # Concatenate items do not exceed capacity.
+          add_all!(underflow) unless underflow.nil?
+
+          # Iteratively replace items, to ensure pseudo-random replacement.
+          overflow.each { |item| replace!(item) } unless overflow.nil?
+        end
+
+        # Stored items are returned and the local buffer is reset.
+        def pop
+          drain!
+        end
+
+        # Return the current number of stored items.
+        def length
+          @items.length
+        end
+
+        # Return if the buffer is empty.
+        def empty?
+          @items.empty?
+        end
+
+        # Closes this buffer, preventing further pushing.
+        # Draining is still allowed.
+        def close
+          @closed = true
+        end
+
+        def closed?
+          @closed
+        end
+
+        protected
+
+        # Segment items into two segments: underflow and overflow.
+        # Underflow are items that will fit into buffer.
+        # Overflow are items that will exceed capacity, after underflow is added.
+        # Returns each array, and nil if there is no underflow/overflow.
+        def overflow_segments(items)
+          underflow = nil
+          overflow = nil
+
+          overflow_size = @max_size > 0 ? (@items.length + items.length) - @max_size : 0
+
+          if overflow_size > 0
+            # Items will overflow
+            if overflow_size < items.length
+              # Partial overflow
+              underflow_end_index = items.length - overflow_size - 1
+              underflow = items[0..underflow_end_index]
+              overflow = items[(underflow_end_index + 1)..-1]
+            else
+              # Total overflow
+              overflow = items
+            end
+          else
+            # Items do not exceed capacity.
+            underflow = items
+          end
+
+          [underflow, overflow]
+        end
+
+        def full?
+          @max_size > 0 && @items.length >= @max_size
+        end
+
+        def add_all!(items)
+          @items.concat(items)
+        end
+
+        def add!(item)
+          @items << item
+        end
+
+        def replace!(item)
+          # Choose random item to be replaced
+          replace_index = rand(@items.length)
+
+          # Replace random item
+          discarded_item = @items[replace_index]
+          @items[replace_index] = item
+
+          # Return discarded item
+          discarded_item
+        end
+
+        def drain!
+          items = @items
+          @items = []
+          items
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/buffer/thread_safe.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require_relative 'random'
+
+module Datadog
+  module Core
+    module Buffer
+      # Buffer that stores objects, has a maximum size, and
+      # can be safely used concurrently on any environment.
+      #
+      # This implementation uses a {Mutex} around public methods, incurring
+      # overhead in order to ensure thread-safety.
+      #
+      # This is implementation is recommended for non-CRuby environments.
+      # If using CRuby, {Datadog::Core::Buffer::CRuby} is a faster implementation with minimal compromise.
+      class ThreadSafe < Random
+        def initialize(max_size)
+          super
+
+          @mutex = Mutex.new
+        end
+
+        # Add a new ``item`` in the local queue. This method doesn't block the execution
+        # even if the buffer is full. In that case, a random item is discarded.
+        def push(item)
+          synchronize { super }
+        end
+
+        def concat(items)
+          synchronize { super }
+        end
+
+        # Return the current number of stored items.
+        def length
+          synchronize { super }
+        end
+
+        # Return if the buffer is empty.
+        def empty?
+          synchronize { super }
+        end
+
+        # Stored items are returned and the local buffer is reset.
+        def pop
+          synchronize { super }
+        end
+
+        def close
+          synchronize { super }
+        end
+
+        def synchronize(&block)
+          @mutex.synchronize(&block)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/chunker.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    # Chunks list of elements into batches
+    module Chunker
+      module_function
+
+      # Chunks a list into batches of at most +max_chunk_size+ elements each.
+      #
+      # An exception can occur if a single element is too large. That single
+      # element will be returned in its own chunk. You have to verify by yourself
+      # when such elements are returned.
+      #
+      # @param list [Enumerable] list of elements
+      # @param max_chunk_size [Numeric] maximum acceptable chunk size
+      # @return [Enumerable] lazy list of chunks
+      def chunk_by_size(list, max_chunk_size)
+        chunk_agg = 0
+        list.slice_before do |elem|
+          size = elem.size
+          chunk_agg += size
+          if chunk_agg > max_chunk_size
+            # Can't fit element in current chunk, start a new one.
+            chunk_agg = size
+            true
+          else
+            # Add to current chunk
+            false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/configuration/agent_settings_resolver.rb

@@ -0,0 +1,352 @@
+# frozen_string_literal: true
+
+require 'uri'
+
+require_relative 'settings'
+require_relative 'ext'
+require_relative '../transport/ext'
+
+module Datadog
+  module Core
+    module Configuration
+      # This class unifies all the different ways that users can configure how we talk to the agent.
+      #
+      # It has quite a lot of complexity, but this complexity just reflects the actual complexity we have around our
+      # configuration today. E.g., this is just all of the complexity regarding agent settings gathered together in a
+      # single place. As we deprecate more and more of the different ways that these things can be configured,
+      # this class will reflect that simplification as well.
+      #
+      # Whenever there is a conflict (different configurations are provided in different orders), it MUST warn the users
+      # about it and pick a value based on the following priority: code > environment variable > defaults.
+      class AgentSettingsResolver
+        AgentSettings = Struct.new(
+          :adapter,
+          :ssl,
+          :hostname,
+          :port,
+          :uds_path,
+          :timeout_seconds,
+          keyword_init: true
+        ) do
+          def initialize(*)
+            super
+            freeze
+          end
+        end
+
+        def self.call(settings, logger: Datadog.logger)
+          new(settings, logger: logger).send(:call)
+        end
+
+        private
+
+        attr_reader \
+          :logger,
+          :settings
+
+        def initialize(settings, logger: Datadog.logger)
+          @settings = settings
+          @logger = logger
+        end
+
+        def call
+          AgentSettings.new(
+            adapter: adapter,
+            ssl: ssl?,
+            hostname: hostname,
+            port: port,
+            uds_path: uds_path,
+            timeout_seconds: timeout_seconds,
+          )
+        end
+
+        def adapter
+          if should_use_uds?
+            Datadog::Core::Configuration::Ext::Agent::UnixSocket::ADAPTER
+          else
+            Datadog::Core::Configuration::Ext::Agent::HTTP::ADAPTER
+          end
+        end
+
+        def configured_hostname
+          return @configured_hostname if defined?(@configured_hostname)
+
+          @configured_hostname = pick_from(
+            DetectedConfiguration.new(
+              friendly_name: "'c.agent.host'",
+              value: settings.agent.host
+            ),
+            DetectedConfiguration.new(
+              friendly_name: "#{Datadog::Core::Configuration::Ext::Agent::ENV_DEFAULT_URL} environment variable",
+              value: parsed_http_url&.hostname
+            ),
+            DetectedConfiguration.new(
+              friendly_name: "#{Datadog::Core::Configuration::Ext::Agent::ENV_DEFAULT_HOST} environment variable",
+              value: ENV[Datadog::Core::Configuration::Ext::Agent::ENV_DEFAULT_HOST]
+            )
+          )
+        end
+
+        def configured_port
+          return @configured_port if defined?(@configured_port)
+
+          @configured_port = pick_from(
+            try_parsing_as_integer(
+              friendly_name: '"c.agent.port"',
+              value: settings.agent.port,
+            ),
+            DetectedConfiguration.new(
+              friendly_name: "#{Datadog::Core::Configuration::Ext::Agent::ENV_DEFAULT_URL} environment variable",
+              value: parsed_http_url&.port,
+            ),
+            try_parsing_as_integer(
+              friendly_name: "#{Datadog::Core::Configuration::Ext::Agent::ENV_DEFAULT_PORT} environment variable",
+              value: ENV[Datadog::Core::Configuration::Ext::Agent::ENV_DEFAULT_PORT],
+            )
+          )
+        end
+
+        def configured_ssl
+          return @configured_ssl if defined?(@configured_ssl)
+
+          @configured_ssl = pick_from(
+            DetectedConfiguration.new(
+              friendly_name: '"c.agent.use_ssl"',
+              value: settings.agent.use_ssl,
+            ),
+            DetectedConfiguration.new(
+              friendly_name: "#{Datadog::Core::Configuration::Ext::Agent::ENV_DEFAULT_URL} environment variable",
+              value: parsed_url_ssl?,
+            )
+          )
+        end
+
+        def configured_timeout_seconds
+          return @configured_timeout_seconds if defined?(@configured_timeout_seconds)
+
+          @configured_timeout_seconds = pick_from(
+            try_parsing_as_integer(
+              friendly_name: '"c.agent.timeout_seconds"',
+              value: settings.agent.timeout_seconds,
+            ),
+            try_parsing_as_integer(
+              friendly_name: "#{Datadog::Core::Configuration::Ext::Agent::ENV_DEFAULT_TIMEOUT_SECONDS} "\
+                'environment variable',
+              value: ENV[Datadog::Core::Configuration::Ext::Agent::ENV_DEFAULT_TIMEOUT_SECONDS],
+            )
+          )
+        end
+
+        def configured_uds_path
+          return @configured_uds_path if defined?(@configured_uds_path)
+
+          @configured_uds_path = pick_from(
+            DetectedConfiguration.new(
+              friendly_name: "'c.agent.uds_path'",
+              value: settings.agent.uds_path
+            ),
+            DetectedConfiguration.new(
+              friendly_name: "#{Datadog::Core::Configuration::Ext::Agent::ENV_DEFAULT_URL} environment variable",
+              value: parsed_url_uds_path
+            )
+          )
+        end
+
+        def parsed_url_ssl?
+          return nil if parsed_url.nil?
+
+          parsed_url.scheme == 'https'
+        end
+
+        def try_parsing_as_integer(value:, friendly_name:)
+          value =
+            begin
+              Integer(value) if value
+            rescue ArgumentError, TypeError
+              log_warning("Invalid value for #{friendly_name} (#{value.inspect}). Ignoring this configuration.")
+
+              nil
+            end
+
+          DetectedConfiguration.new(friendly_name: friendly_name, value: value)
+        end
+
+        def ssl?
+          if should_use_uds?
+            false
+          else
+            configured_ssl || Datadog::Core::Configuration::Ext::Agent::HTTP::DEFAULT_USE_SSL
+          end
+        end
+
+        def hostname
+          configured_hostname || (should_use_uds? ? nil : Datadog::Core::Configuration::Ext::Agent::HTTP::DEFAULT_HOST)
+        end
+
+        def port
+          configured_port || (should_use_uds? ? nil : Datadog::Core::Configuration::Ext::Agent::HTTP::DEFAULT_PORT)
+        end
+
+        def timeout_seconds
+          return configured_timeout_seconds unless configured_timeout_seconds.nil?
+
+          if should_use_uds?
+            Datadog::Core::Configuration::Ext::Agent::UnixSocket::DEFAULT_TIMEOUT_SECONDS
+          else
+            Datadog::Core::Configuration::Ext::Agent::HTTP::DEFAULT_TIMEOUT_SECONDS
+          end
+        end
+
+        def parsed_url_uds_path
+          return nil unless parsed_url && unix_scheme?(parsed_url)
+
+          path = parsed_url.to_s
+          # Some versions of the built-in uri gem leave the original url untouched, and others remove the //, so this
+          # supports both
+          if path.start_with?('unix://')
+            path.sub('unix://', '')
+          else
+            path.sub('unix:', '')
+          end
+        end
+
+        # Unix socket path in the file system
+        def uds_path
+          return nil unless should_use_uds?
+
+          configured_uds_path || uds_fallback
+        end
+
+        # We only use the default unix socket if it is already present.
+        # This is by design, as we still want to use the default host:port if no unix socket is present.
+        def uds_fallback
+          return @uds_fallback if defined?(@uds_fallback)
+
+          @uds_fallback =
+            if configured_hostname.nil? &&
+                configured_port.nil? &&
+                File.exist?(Datadog::Core::Configuration::Ext::Agent::UnixSocket::DEFAULT_PATH)
+
+              Datadog::Core::Configuration::Ext::Agent::UnixSocket::DEFAULT_PATH
+            end
+        end
+
+        def should_use_uds?
+          can_use_uds? && !mixed_http_and_uds?
+        end
+
+        def can_use_uds?
+          !configured_uds_path.nil? ||
+            # If no agent settings have been provided, we try to connect using a local unix socket.
+            # We only do so if the socket is present when `datadog` runs.
+            !uds_fallback.nil?
+        end
+
+        def parsed_url
+          return @parsed_url if defined?(@parsed_url)
+
+          unparsed_url_from_env = ENV[Datadog::Core::Configuration::Ext::Agent::ENV_DEFAULT_URL]
+
+          @parsed_url =
+            if unparsed_url_from_env
+              parsed = URI.parse(unparsed_url_from_env)
+
+              if http_scheme?(parsed) || unix_scheme?(parsed)
+                parsed
+              else
+                # rubocop:disable Layout/LineLength
+                log_warning(
+                  "Invalid URI scheme '#{parsed.scheme}' for #{Datadog::Core::Configuration::Ext::Agent::ENV_DEFAULT_URL} " \
+                  "environment variable ('#{unparsed_url_from_env}'). " \
+                  "Ignoring the contents of #{Datadog::Core::Configuration::Ext::Agent::ENV_DEFAULT_URL}."
+                )
+                # rubocop:enable Layout/LineLength
+
+                nil
+              end
+            end
+        end
+
+        def pick_from(*configurations_in_priority_order)
+          detected_configurations_in_priority_order = configurations_in_priority_order.select(&:value?)
+
+          if detected_configurations_in_priority_order.any?
+            warn_if_configuration_mismatch(detected_configurations_in_priority_order)
+
+            # The configurations are listed in priority, so we only need to look at the first; if there's more than
+            # one, we emit a warning above
+            detected_configurations_in_priority_order.first.value
+          end
+        end
+
+        def warn_if_configuration_mismatch(detected_configurations_in_priority_order)
+          return unless detected_configurations_in_priority_order.map(&:value).uniq.size > 1
+
+          log_warning(
+            'Configuration mismatch: values differ between ' \
+            "#{detected_configurations_in_priority_order
+              .map { |config| "#{config.friendly_name} (#{config.value.inspect})" }.join(' and ')}" \
+            ". Using #{detected_configurations_in_priority_order.first.value.inspect} and ignoring other configuration."
+          )
+        end
+
+        def log_warning(message)
+          logger.warn(message) if logger
+        end
+
+        def http_scheme?(uri)
+          ['http', 'https'].include?(uri.scheme)
+        end
+
+        # Expected to return nil (not false!) when it's not http
+        def parsed_http_url
+          parsed_url if parsed_url && http_scheme?(parsed_url)
+        end
+
+        def unix_scheme?(uri)
+          uri.scheme == 'unix'
+        end
+
+        # When we have mixed settings for http/https and uds, we print a warning and ignore the uds settings
+        def mixed_http_and_uds?
+          return @mixed_http_and_uds if defined?(@mixed_http_and_uds)
+
+          @mixed_http_and_uds = (configured_hostname || configured_port) && can_use_uds?
+
+          if @mixed_http_and_uds
+            warn_if_configuration_mismatch(
+              [
+                DetectedConfiguration.new(
+                  friendly_name: 'configuration of hostname/port for http/https use',
+                  value: "hostname: '#{hostname}', port: '#{port}'",
+                ),
+                DetectedConfiguration.new(
+                  friendly_name: 'configuration for unix domain socket',
+                  value: parsed_url.to_s,
+                ),
+              ]
+            )
+          end
+
+          @mixed_http_and_uds
+        end
+
+        # Represents a given configuration value and where we got it from
+        class DetectedConfiguration
+          attr_reader :friendly_name, :value
+
+          def initialize(friendly_name:, value:)
+            @friendly_name = friendly_name
+            @value = value
+            freeze
+          end
+
+          def value?
+            !value.nil?
+          end
+        end
+        private_constant :DetectedConfiguration
+      end
+    end
+  end
+end

data/lib/datadog/core/configuration/base.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require_relative 'options'
+
+module Datadog
+  module Core
+    module Configuration
+      # Basic configuration behavior
+      # @public_api
+      module Base
+        def self.included(base)
+          base.include(Options)
+
+          base.extend(ClassMethods)
+          base.include(InstanceMethods)
+        end
+
+        # Class methods for configuration
+        # @public_api
+        module ClassMethods
+          protected
+
+          # Allows subgroupings of settings to be defined.
+          # e.g. `settings :foo { option :bar }` --> `config.foo.bar`
+          # @param [Symbol] name option name. Methods will be created based on this name.
+          def settings(name, &block)
+            settings_class = new_settings_class(name, &block)
+
+            option(name) do |o|
+              o.default { settings_class.new }
+
+              o.resetter do |value|
+                value.reset! if value.respond_to?(:reset!)
+                value
+              end
+            end
+
+            settings_class
+          end
+
+          private
+
+          def new_settings_class(name, &block)
+            Class.new { include Configuration::Base }.tap do |klass|
+              klass.instance_variable_set(:@settings_name, name)
+              klass.instance_eval(&block) if block
+            end
+          end
+        end
+
+        # Instance methods for configuration
+        # @public_api
+        module InstanceMethods
+          def initialize(options = {})
+            configure(options) unless options.empty?
+          end
+
+          def configure(opts = {})
+            opts.each do |name, value|
+              if respond_to?("#{name}=")
+                send("#{name}=", value)
+              elsif option_defined?(name)
+                set_option(name, value)
+              end
+            end
+
+            # Apply any additional settings from block
+            yield(self) if block_given?
+          end
+
+          def to_h
+            options_hash
+          end
+
+          # Retrieves a nested option from a list of symbols
+          def dig(*options)
+            raise ArgumentError, 'expected at least one option' if options.empty?
+
+            options.inject(self) do |receiver, option|
+              receiver.send(option)
+            end
+          end
+
+          def reset!
+            reset_options!
+          end
+        end
+      end
+    end
+  end
+end