datadog 2.0.0.beta1

data/lib/datadog/core/configuration.rb

@@ -0,0 +1,296 @@
+# frozen_string_literal: true
+
+require_relative 'configuration/components'
+require_relative 'configuration/settings'
+require_relative 'telemetry/emitter'
+require_relative 'logger'
+require_relative 'pin'
+
+module Datadog
+  module Core
+    # Configuration provides a unique access point for configurations
+    module Configuration
+      # Used to ensure that @components initialization/reconfiguration is performed one-at-a-time, by a single thread.
+      #
+      # This is important because components can end up being accessed from multiple application threads (for instance on
+      # a threaded webserver), and we don't want their initialization to clash (for instance, starting two profilers...).
+      #
+      # Note that a Mutex **IS NOT** reentrant: the same thread cannot grab the same Mutex more than once.
+      # This means below we are careful not to nest calls to methods that would trigger initialization and grab the lock.
+      #
+      # Every method that directly or indirectly mutates @components should be holding the lock (through
+      # #safely_synchronize) while doing so.
+      COMPONENTS_WRITE_LOCK = Mutex.new
+      private_constant :COMPONENTS_WRITE_LOCK
+
+      # We use a separate lock when reading the @components, so that they continue to be accessible during reconfiguration.
+      # This was needed because we ran into several issues where we still needed to read the old
+      # components while the COMPONENTS_WRITE_LOCK was being held (see https://github.com/DataDog/dd-trace-rb/pull/1387
+      # and https://github.com/DataDog/dd-trace-rb/pull/1373#issuecomment-799593022 ).
+      #
+      # Technically on MRI we could get away without this lock, but on non-MRI Rubies, we may run into issues because
+      # we fall into the "UnsafeDCLFactory" case of https://shipilev.net/blog/2014/safe-public-construction/ .
+      # Specifically, on JRuby reads from the @components do NOT have volatile semantics, and on TruffleRuby they do
+      # BUT just as an implementation detail, see https://github.com/jruby/jruby/wiki/Concurrency-in-jruby#volatility and
+      # https://github.com/DataDog/dd-trace-rb/pull/1329#issuecomment-776750377 .
+      # Concurrency is hard.
+      COMPONENTS_READ_LOCK = Mutex.new
+      private_constant :COMPONENTS_READ_LOCK
+
+      attr_writer :configuration
+
+      # Current Datadog configuration.
+      #
+      # Access to non-global configuration will raise an error.
+      #
+      # To modify the configuration, use {.configure}.
+      #
+      # @return [Datadog::Core::Configuration::Settings]
+      # @!attribute [r] configuration
+
+      # @public_api
+      def configuration
+        @configuration ||= Settings.new
+      end
+
+      # Apply global configuration changes to `Datadog`. An example of a {.configure} call:
+      #
+      # ```
+      # Datadog.configure do |c|
+      #   c.service = 'my-service'
+      #   c.env = 'staging'
+      #   # c.diagnostics.debug = true # Enables debug output
+      # end
+      # ```
+      #
+      # See {Datadog::Core::Configuration::Settings} for all available options, defaults, and
+      # available environment variables for configuration.
+      #
+      # Only permits access to global configuration settings; others will raise an error.
+      # If you wish to configure a setting for a specific Datadog component (e.g. Tracing),
+      # use the corresponding `Datadog::COMPONENT.configure` method instead.
+      #
+      # Because many configuration changes require restarting internal components,
+      # invoking {.configure} is the only safe way to change `Datadog` configuration.
+      #
+      # Successive calls to {.configure} maintain the previous configuration values:
+      # configuration is additive between {.configure} calls.
+      #
+      # The yielded configuration `c` comes pre-populated from environment variables, if
+      # any are applicable.
+      #
+      # @yieldparam [Datadog::Core::Configuration::Settings] c the mutable configuration object
+      def configure
+        configuration = self.configuration
+        yield(configuration)
+
+        safely_synchronize do |write_components|
+          write_components.call(
+            if components?
+              replace_components!(configuration, @components)
+            else
+              components = build_components(configuration)
+              components.telemetry.started!
+              components
+            end
+          )
+        end
+
+        configuration
+      end
+
+      # Apply configuration changes only to a specific Ruby object.
+      #
+      # Certain integrations or Datadog features may use these
+      # settings to customize behavior for this object.
+      #
+      # An example of a {.configure_onto} call:
+      #
+      # ```
+      # client = Net::HTTP.new(host, port)
+      # Datadog.configure_onto(client, service_name: 'api-requests', split_by_domain: true)
+      # ```
+      #
+      # In this example, it will configure the `client` object with custom options
+      # `service_name: 'api-requests', split_by_domain: true`. The `Net::HTTP` integration
+      # will then use these customized options when the `client` is used, whereas other
+      # clients will use the `service_name: 'http-requests'` configuration provided to the
+      # `Datadog.configure` call block.
+      #
+      # {.configure_onto} is used to separate cases where spans generated by certain objects
+      # require exceptional options.
+      #
+      # The configuration keyword arguments provided should match well known options defined
+      # in the integration or feature that would use them.
+      #
+      # For example, for `Datadog.configure_onto(redis_client, **opts)`, `opts` can be
+      # any of the options in the Redis {Datadog::Tracing::Contrib::Redis::Configuration::Settings} class.
+      #
+      # @param [Object] target the object to receive configuration options
+      # @param [Hash] opts keyword arguments respective to the integration this object belongs to
+      # @public_api
+      def configure_onto(target, **opts)
+        Pin.set_on(target, **opts)
+      end
+
+      # Get configuration changes applied only to a specific Ruby object, via {.configure_onto}.
+      # An example of an object with specific configuration:
+      #
+      # ```
+      # client = Net::HTTP.new(host, port)
+      # Datadog.configure_onto(client, service_name: 'api-requests', split_by_domain: true)
+      # config = Datadog.configuration_for(client)
+      # config[:service_name] # => 'api-requests'
+      # config[:split_by_domain] # => true
+      # ```
+      #
+      # @param [Object] target the object to receive configuration options
+      # @param [Object] option an option to retrieve from the object configuration
+      # @public_api
+      def configuration_for(target, option = nil)
+        pin = Pin.get_from(target)
+        return pin unless option
+
+        pin[option] if pin
+      end
+
+      # Internal {Datadog::Statsd} metrics collection.
+      #
+      # The list of metrics collected can be found in {Datadog::Core::Diagnostics::Ext::Health::Metrics}.
+      # @public_api
+      def health_metrics
+        components.health_metrics
+      end
+
+      def logger
+        # avoid initializing components if they didn't already exist
+        current_components = components(allow_initialization: false)
+
+        if current_components
+          @temp_logger = nil
+          current_components.logger
+        else
+          logger_without_components
+        end
+      end
+
+      # Gracefully shuts down all components.
+      #
+      # Components will still respond to method calls as usual,
+      # but might not internally perform their work after shutdown.
+      #
+      # This avoids errors being raised across the host application
+      # during shutdown, while allowing for graceful decommission of resources.
+      #
+      # Components won't be automatically reinitialized after a shutdown.
+      def shutdown!
+        safely_synchronize do
+          @components.shutdown! if components?
+        end
+      end
+
+      protected
+
+      def components(allow_initialization: true)
+        current_components = COMPONENTS_READ_LOCK.synchronize { defined?(@components) && @components }
+        return current_components if current_components || !allow_initialization
+
+        safely_synchronize do |write_components|
+          (defined?(@components) && @components) || write_components.call(build_components(configuration))
+        end
+      end
+
+      private
+
+      # Gracefully shuts down Datadog components and disposes of component references,
+      # allowing execution to start anew.
+      #
+      # In contrast with +#shutdown!+, components will be automatically
+      # reinitialized after a reset.
+      #
+      # Used internally to ensure a clean environment between test runs.
+      def reset!
+        safely_synchronize do |write_components|
+          if components?
+            @components.shutdown!
+            @temp_logger = nil # Reset to ensure instance and log level are reset for next run
+          end
+
+          write_components.call(nil)
+          configuration.reset!
+        end
+      end
+
+      def safely_synchronize
+        # Writes to @components should only happen through this proc. Because this proc is only accessible to callers of
+        # safely_synchronize, this forces all writers to go through this method.
+        write_components = proc do |new_value|
+          COMPONENTS_READ_LOCK.synchronize { @components = new_value }
+        end
+
+        COMPONENTS_WRITE_LOCK.synchronize do
+          begin
+            yield write_components
+          rescue ThreadError => e
+            logger_without_components.error(
+              'Detected deadlock during datadog initialization. ' \
+              'Please report this at https://github.com/DataDog/dd-trace-rb/blob/master/CONTRIBUTING.md#found-a-bug' \
+              "\n\tSource:\n\t#{Array(e.backtrace).join("\n\t")}"
+            )
+            nil
+          end
+        end
+      end
+
+      def components?
+        # This does not need to grab the COMPONENTS_READ_LOCK because it's not returning the components
+        (defined?(@components) && @components) != nil
+      end
+
+      def build_components(settings)
+        components = Components.new(settings)
+        components.startup!(settings)
+        components
+      end
+
+      def replace_components!(settings, old)
+        components = Components.new(settings)
+
+        old.shutdown!(components)
+        components.startup!(settings)
+        components
+      end
+
+      def logger_without_components
+        # Use default logger without initializing components.
+        # This enables logging during initialization, otherwise we'd run into deadlocks.
+        @temp_logger ||= begin
+          logger = configuration.logger.instance || Core::Logger.new($stdout)
+          logger.level = configuration.diagnostics.debug ? ::Logger::DEBUG : configuration.logger.level
+          logger
+        end
+      end
+
+      # Called from our at_exit hook whenever there was a pending Interrupt exception (e.g. typically due to ctrl+c)
+      # to print a nice message whenever we're taking a bit longer than usual to finish the process.
+      def handle_interrupt_shutdown!
+        logger = Datadog.logger
+        shutdown_thread = Thread.new { shutdown! }
+        unless Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.3')
+          shutdown_thread.name = Datadog::Core::Configuration.name
+        end
+
+        print_message_treshold_seconds = 0.2
+
+        slow_shutdown = shutdown_thread.join(print_message_treshold_seconds).nil?
+
+        if slow_shutdown
+          logger.info 'Reporting remaining data... Press ctrl+c to exit immediately.'
+          shutdown_thread.join
+        end
+
+        nil
+      end
+    end
+  end
+end

data/lib/datadog/core/diagnostics/environment_logger.rb

@@ -0,0 +1,173 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'rbconfig'
+require 'time'
+
+module Datadog
+  module Core
+    module Diagnostics
+      # Base class for EnvironmentLoggers - should allow for easy reporting by users to Datadog support.
+      module EnvironmentLogging
+        def log_configuration!(prefix, data)
+          logger.info("DATADOG CONFIGURATION - #{prefix} - #{data}")
+        end
+
+        def log_error!(prefix, type, error)
+          logger.warn("DATADOG ERROR - #{prefix} - #{type}: #{error}")
+        end
+
+        protected
+
+        def logger
+          Datadog.logger
+        end
+
+        # Are we logging the environment data?
+        def log?
+          startup_logs_enabled = Datadog.configuration.diagnostics.startup_logs.enabled
+          if startup_logs_enabled.nil?
+            !repl? && !rspec? # Suppress logs if we are running in a REPL or rspec
+          else
+            startup_logs_enabled
+          end
+        end
+
+        REPL_PROGRAM_NAMES = %w[irb pry].freeze
+
+        def repl?
+          REPL_PROGRAM_NAMES.include?($PROGRAM_NAME)
+        end
+
+        def rspec?
+          $PROGRAM_NAME.end_with?('rspec')
+        end
+      end
+
+      # Collects and logs Core diagnostic information
+      module EnvironmentLogger
+        extend EnvironmentLogging
+
+        def self.collect_and_log!(extra_fields = nil)
+          if log?
+            data = EnvironmentCollector.collect_config!
+            data = data.merge(extra_fields) if extra_fields
+            log_configuration!('CORE', data.to_json)
+          end
+        rescue => e
+          logger.warn("Failed to collect core environment information: #{e} Location: #{Array(e.backtrace).first}")
+        end
+      end
+
+      # Collects environment information for Core diagnostic logging
+      module EnvironmentCollector
+        class << self
+          def collect_config!
+            {
+              date: date,
+              os_name: os_name,
+              version: version,
+              lang: lang,
+              lang_version: lang_version,
+              env: env,
+              service: service,
+              dd_version: dd_version,
+              debug: debug,
+              tags: tags,
+              runtime_metrics_enabled: runtime_metrics_enabled,
+              vm: vm,
+              health_metrics_enabled: health_metrics_enabled
+            }
+          end
+
+          # @return [String] current time in ISO8601 format
+          def date
+            Time.now.utc.iso8601
+          end
+
+          # Best portable guess of OS information.
+          # @return [String] platform string
+          def os_name
+            RbConfig::CONFIG['host']
+          end
+
+          # @return [String] datadog version
+          def version
+            Datadog::VERSION::STRING
+          end
+
+          # @return [String] "ruby"
+          def lang
+            Core::Environment::Ext::LANG
+          end
+
+          # Supported Ruby language version.
+          # Will be distinct from VM version for non-MRI environments.
+          # @return [String]
+          def lang_version
+            Core::Environment::Ext::LANG_VERSION
+          end
+
+          # @return [String] configured application environment
+          def env
+            Datadog.configuration.env
+          end
+
+          # @return [String] configured application service name
+          def service
+            Datadog.configuration.service
+          end
+
+          # @return [String] configured application version
+          def dd_version
+            Datadog.configuration.version
+          end
+
+          # @return [Boolean, nil] debug mode enabled in configuration
+          def debug
+            !!Datadog.configuration.diagnostics.debug
+          end
+
+          # @return [Hash, nil] concatenated list of global tracer tags configured
+          def tags
+            tags = Datadog.configuration.tags
+            return nil if tags.empty?
+
+            hash_serializer(tags)
+          end
+
+          # @return [Boolean, nil] runtime metrics enabled in configuration
+          def runtime_metrics_enabled
+            Datadog.configuration.runtime_metrics.enabled
+          end
+
+          # Ruby VM name and version.
+          # Examples: "ruby-2.7.1", "jruby-9.2.11.1", "truffleruby-20.1.0"
+          # @return [String, nil]
+          def vm
+            # RUBY_ENGINE_VERSION returns the VM version, which
+            # will differ from RUBY_VERSION for non-mri VMs.
+            if defined?(RUBY_ENGINE_VERSION)
+              "#{RUBY_ENGINE}-#{RUBY_ENGINE_VERSION}"
+            else
+              # Ruby < 2.3 doesn't support RUBY_ENGINE_VERSION
+              "#{RUBY_ENGINE}-#{RUBY_VERSION}"
+            end
+          end
+
+          # @return [Boolean, nil] health metrics enabled in configuration
+          def health_metrics_enabled
+            !!Datadog.configuration.health_metrics.enabled
+          end
+
+          private
+
+          # Outputs "k1:v1,k2:v2,..."
+          def hash_serializer(h)
+            h.map { |k, v| "#{k}:#{v}" }.join(',')
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/diagnostics/health.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require_relative '../metrics/client'
+require_relative '../../tracing/diagnostics/health'
+
+module Datadog
+  module Core
+    module Diagnostics
+      # Health-related diagnostics
+      module Health
+        # Health metrics for diagnostics
+        class Metrics < Core::Metrics::Client
+          # TODO: Don't reference this. Have tracing add its metrics behavior.
+          extend Tracing::Diagnostics::Health::Metrics
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/encoding.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'msgpack'
+
+module Datadog
+  module Core
+    # Encoding module that encodes data for the AgentTransport
+    module Encoding
+      # Encoder interface that provides the logic to encode traces and service
+      # @abstract
+      module Encoder
+        def content_type
+          raise NotImplementedError
+        end
+
+        # Concatenates a list of elements previously encoded by +#encode+.
+        def join(encoded_elements)
+          raise NotImplementedError
+        end
+
+        # Serializes a single trace into a String suitable for network transmission.
+        def encode(_)
+          raise NotImplementedError
+        end
+      end
+
+      # Encoder for the JSON format
+      module JSONEncoder
+        extend Encoder
+
+        CONTENT_TYPE = 'application/json'
+
+        module_function
+
+        def content_type
+          CONTENT_TYPE
+        end
+
+        def encode(obj)
+          JSON.dump(obj)
+        end
+
+        def join(encoded_data)
+          "[#{encoded_data.join(',')}]"
+        end
+      end
+
+      # Encoder for the Msgpack format
+      module MsgpackEncoder
+        extend Encoder
+
+        module_function
+
+        CONTENT_TYPE = 'application/msgpack'
+
+        def content_type
+          CONTENT_TYPE
+        end
+
+        def encode(obj)
+          MessagePack.pack(obj)
+        end
+
+        def join(encoded_data)
+          packer = MessagePack::Packer.new
+          packer.write_array_header(encoded_data.size)
+
+          (packer.buffer.to_a + encoded_data).join
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/environment/cgroup.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require_relative 'ext'
+
+module Datadog
+  module Core
+    module Environment
+      # Reads information from Linux cgroups.
+      # This information is used to extract information
+      # about the current Linux container identity.
+      # @see https://man7.org/linux/man-pages/man7/cgroups.7.html
+      module Cgroup
+        LINE_REGEX = /^(\d+):([^:]*):(.+)$/.freeze
+
+        Descriptor = Struct.new(
+          :id,
+          :groups,
+          :path,
+          :controllers
+        )
+
+        module_function
+
+        def descriptors(process = 'self')
+          [].tap do |descriptors|
+            begin
+              filepath = "/proc/#{process}/cgroup"
+
+              if File.exist?(filepath)
+                File.foreach("/proc/#{process}/cgroup") do |line|
+                  line = line.strip
+                  descriptors << parse(line) unless line.empty?
+                end
+              end
+            rescue StandardError => e
+              Datadog.logger.error(
+                "Error while parsing cgroup. Cause: #{e.class.name} #{e.message} Location: #{Array(e.backtrace).first}"
+              )
+            end
+          end
+        end
+
+        def parse(line)
+          id, groups, path = line.scan(LINE_REGEX).first
+
+          Descriptor.new(id, groups, path).tap do |descriptor|
+            descriptor.controllers = groups.split(',') unless groups.nil?
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/environment/class_count.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Environment
+      # Retrieves number of classes from runtime
+      module ClassCount
+        def self.value
+          ::ObjectSpace.count_objects[:T_CLASS]
+        end
+
+        def self.available?
+          return @class_count_available if defined?(@class_count_available)
+
+          @class_count_available =
+            ::ObjectSpace.respond_to?(:count_objects) && ::ObjectSpace.count_objects.key?(:T_CLASS)
+        end
+      end
+    end
+  end
+end