datadog 2.3.0 → 2.5.0

data/lib/datadog/core/rate_limiter.rb

@@ -0,0 +1,183 @@
+# frozen_string_literal: true
+
+require_relative 'utils/time'
+
+module Datadog
+  module Core
+    # Checks for rate limiting on a resource.
+    class RateLimiter
+      # Checks if resource of specified size can be
+      # conforms with the current limit.
+      #
+      # Implementations of this method are not guaranteed
+      # to be side-effect free.
+      #
+      # @return [Boolean] whether a resource conforms with the current limit
+      def allow?(size = 1); end
+
+      # The effective rate limiting ratio based on
+      # recent calls to `allow?`.
+      #
+      # @return [Float] recent allowance ratio
+      def effective_rate; end
+    end
+
+    # Implementation of the Token Bucket metering algorithm
+    # for rate limiting.
+    #
+    # @see https://en.wikipedia.org/wiki/Token_bucket Token bucket
+    class TokenBucket < RateLimiter
+      attr_reader :rate, :max_tokens
+
+      # @param rate [Numeric] Allowance rate, in units per second
+      #  if rate is negative, always allow
+      #  if rate is zero, never allow
+      # @param max_tokens [Numeric] Limit of available tokens
+      def initialize(rate, max_tokens = rate)
+        super()
+
+        raise ArgumentError, "rate must be a number: #{rate}" unless rate.is_a?(Numeric)
+        raise ArgumentError, "max_tokens must be a number: #{max_tokens}" unless max_tokens.is_a?(Numeric)
+
+        @rate = rate
+        @max_tokens = max_tokens
+
+        @tokens = max_tokens
+        @total_messages = 0
+        @conforming_messages = 0
+        @prev_conforming_messages = nil
+        @prev_total_messages = nil
+        @current_window = nil
+
+        @last_refill = Core::Utils::Time.get_time
+      end
+
+      # Checks if a message of provided +size+
+      # conforms with the current bucket limit.
+      #
+      # If it does, return +true+ and remove +size+
+      # tokens from the bucket.
+      # If it does not, return +false+ without affecting
+      # the tokens from the bucket.
+      #
+      # @return [Boolean] +true+ if message conforms with current bucket limit
+      def allow?(size = 1)
+        allowed = should_allow?(size)
+        update_rate_counts(allowed)
+        allowed
+      end
+
+      # Ratio of 'conformance' per 'total messages' checked
+      # averaged for the past 2 buckets
+      #
+      # Returns +1.0+ when no messages have been checked yet.
+      #
+      # @return [Float] Conformance ratio, between +[0,1]+
+      def effective_rate
+        return 0.0 if @rate.zero?
+        return 1.0 if @rate < 0 || @total_messages.zero?
+
+        return current_window_rate if @prev_conforming_messages.nil? || @prev_total_messages.nil?
+
+        (@conforming_messages.to_f + @prev_conforming_messages.to_f) / (@total_messages + @prev_total_messages)
+      end
+
+      # Ratio of 'conformance' per 'total messages' checked
+      # on this bucket
+      #
+      # Returns +1.0+ when no messages have been checked yet.
+      #
+      # @return [Float] Conformance ratio, between +[0,1]+
+      def current_window_rate
+        return 1.0 if @total_messages.zero?
+
+        @conforming_messages.to_f / @total_messages
+      end
+
+      # @return [Numeric] number of tokens currently available
+      def available_tokens
+        @tokens
+      end
+
+      private
+
+      def refill_since_last_message
+        now = Core::Utils::Time.get_time
+        elapsed = now - @last_refill
+
+        # Update the number of available tokens, but ensure we do not exceed the max
+        # we return the min of tokens + rate*elapsed, or max tokens
+        refill_tokens(@rate * elapsed)
+
+        @last_refill = now
+      end
+
+      def refill_tokens(size)
+        @tokens += size
+        @tokens = @max_tokens if @tokens > @max_tokens
+      end
+
+      def increment_total_count
+        @total_messages += 1
+      end
+
+      def increment_conforming_count
+        @conforming_messages += 1
+      end
+
+      def should_allow?(size = 1)
+        # rate limit of 0 blocks everything
+        return false if @rate.zero?
+
+        # negative rate limit disables rate limiting
+        return true if @rate < 0
+
+        refill_since_last_message
+
+        # if tokens < 1 we don't allow?
+        return false if @tokens < size
+
+        @tokens -= size
+
+        true
+      end
+
+      # Sets and Updates the past two 1 second windows for which
+      # the rate limiter must compute it's rate over and updates
+      # the total count, and conforming message count if +allowed+
+      def update_rate_counts(allowed)
+        now = Core::Utils::Time.get_time
+
+        # No tokens have been seen yet, start a new window
+        if @current_window.nil?
+          @current_window = now
+        # If more than 1 second has past since last window, reset
+        elsif now - @current_window >= 1
+          @prev_conforming_messages = @conforming_messages
+          @prev_total_messages = @total_messages
+          @conforming_messages = 0
+          @total_messages = 0
+          @current_window = now
+        end
+
+        increment_conforming_count if allowed
+
+        increment_total_count
+      end
+    end
+
+    # {Datadog::Core::RateLimiter} that accepts all resources,
+    # with no limits.
+    class UnlimitedLimiter < RateLimiter
+      # @return [Boolean] always +true+
+      def allow?(_ = 1)
+        true
+      end
+
+      # @return [Float] always 100%
+      def effective_rate
+        1.0
+      end
+    end
+  end
+end

data/lib/datadog/core/remote/client/capabilities.rb

@@ -12,10 +12,11 @@ module Datadog
         class Capabilities
           attr_reader :products, :capabilities, :receivers, :base64_capabilities
 
-          def initialize(settings)
+          def initialize(settings, telemetry)
             @capabilities = []
             @products = []
             @receivers = []
+            @telemetry = telemetry
 
             register(settings)
 
@@ -28,12 +29,12 @@ module Datadog
             if settings.respond_to?(:appsec) && settings.appsec.enabled
               register_capabilities(Datadog::AppSec::Remote.capabilities)
               register_products(Datadog::AppSec::Remote.products)
-              register_receivers(Datadog::AppSec::Remote.receivers)
+              register_receivers(Datadog::AppSec::Remote.receivers(@telemetry))
             end
 
             register_capabilities(Datadog::Tracing::Remote.capabilities)
             register_products(Datadog::Tracing::Remote.products)
-            register_receivers(Datadog::Tracing::Remote.receivers)
+            register_receivers(Datadog::Tracing::Remote.receivers(@telemetry))
           end
 
           def register_capabilities(capabilities)

data/lib/datadog/core/remote/component.rb

@@ -39,6 +39,7 @@ module Datadog
               @client.sync
               @healthy ||= true
             rescue Client::SyncError => e
+              # Transient errors due to network or agent. Logged the error but not via telemetry
               Datadog.logger.error do
                 "remote worker client sync error: #{e.message} location: #{Array(e.backtrace).first}. skipping sync"
               end
@@ -48,6 +49,7 @@ module Datadog
               # negotiation object stores error logging state that should be reset.
               negotiation = Negotiation.new(settings, agent_settings)
 
+              # Transient errors due to network or agent. Logged the error but not via telemetry
               Datadog.logger.error do
                 "remote worker error: #{e.class.name} #{e.message} location: #{Array(e.backtrace).first}. "\
                 'reseting client state'
@@ -150,10 +152,10 @@ module Datadog
           #
           # Those checks are instead performed inside the worker loop.
           # This allows users to upgrade their agent while keeping their application running.
-          def build(settings, agent_settings)
+          def build(settings, agent_settings, telemetry:)
             return unless settings.remote.enabled
 
-            new(settings, Client::Capabilities.new(settings), agent_settings)
+            new(settings, Client::Capabilities.new(settings, telemetry), agent_settings)
           end
         end
       end

data/lib/datadog/core/remote/negotiation.rb

@@ -20,7 +20,7 @@ module Datadog
 
           if res.internal_error? && network_error?(res.error)
             unless @logged[:agent_unreachable]
-              Datadog.logger.error { "agent unreachable: cannot negotiate #{path}" }
+              Datadog.logger.warn { "agent unreachable: cannot negotiate #{path}" }
               @logged[:agent_unreachable] = true
             end
 
@@ -29,7 +29,7 @@ module Datadog
 
           if res.not_found?
             unless @logged[:no_info_endpoint]
-              Datadog.logger.error { "agent reachable but has no /info endpoint: cannot negotiate #{path}" }
+              Datadog.logger.warn { "agent reachable but has no /info endpoint: cannot negotiate #{path}" }
               @logged[:no_info_endpoint] = true
             end
 
@@ -38,7 +38,7 @@ module Datadog
 
           unless res.ok?
             unless @logged[:unexpected_response]
-              Datadog.logger.error { "agent reachable but unexpected response: cannot negotiate #{path}" }
+              Datadog.logger.warn { "agent reachable but unexpected response: cannot negotiate #{path}" }
               @logged[:unexpected_response] = true
             end
 
@@ -47,7 +47,7 @@ module Datadog
 
           unless res.endpoints.include?(path)
             unless @logged[:no_config_endpoint]
-              Datadog.logger.error { "agent reachable but does not report #{path}" }
+              Datadog.logger.warn { "agent reachable but does not report #{path}" }
               @logged[:no_config_endpoint] = true
             end
 

data/lib/datadog/core/remote/tie.rb

@@ -19,6 +19,8 @@ module Datadog
             barrier = Datadog::Core::Remote.active_remote.barrier(:once)
           end
 
+          # steep does not permit the next line due to
+          # https://github.com/soutaro/steep/issues/1231
           Boot.new(barrier, t)
         end
       end

data/lib/datadog/core/remote/transport/http.rb

@@ -120,6 +120,11 @@ module Datadog
               # Add container ID, if present.
               container_id = Datadog::Core::Environment::Container.container_id
               headers[Datadog::Core::Transport::Ext::HTTP::HEADER_CONTAINER_ID] = container_id unless container_id.nil?
+              # Sending this header to the agent will disable metrics computation (and billing) on the agent side
+              # by pretending it has already been done on the library side.
+              if Datadog.configuration.appsec.standalone.enabled
+                headers[Datadog::Core::Transport::Ext::HTTP::HEADER_CLIENT_COMPUTED_STATS] = 'yes'
+              end
             end
           end
 

data/lib/datadog/core/remote/worker.rb

@@ -34,7 +34,7 @@ module Datadog
           @starting = true
 
           thread = Thread.new { poll(@interval) }
-          thread.name = self.class.name unless Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.3')
+          thread.name = self.class.name
           thread.thread_variable_set(:fork_safe, true)
           @thr = thread
 

data/lib/datadog/core/runtime/ext.rb

@@ -31,6 +31,7 @@ module Datadog
           METRIC_YJIT_OBJECT_SHAPE_COUNT = 'runtime.ruby.yjit.object_shape_count'
           METRIC_YJIT_OUTLINED_CODE_SIZE = 'runtime.ruby.yjit.outlined_code_size'
           METRIC_YJIT_YJIT_ALLOC_SIZE = 'runtime.ruby.yjit.yjit_alloc_size'
+          METRIC_YJIT_RATIO_IN_YJIT = 'runtime.ruby.yjit.ratio_in_yjit'
 
           TAG_SERVICE = 'service'
         end

data/lib/datadog/core/runtime/metrics.rb

@@ -94,7 +94,7 @@ module Datadog
         def try_flush
           yield
         rescue StandardError => e
-          Datadog.logger.error("Error while sending runtime metric. Cause: #{e.class.name} #{e.message}")
+          Datadog.logger.warn("Error while sending runtime metric. Cause: #{e.class.name} #{e.message}")
         end
 
         def default_metric_options
@@ -181,6 +181,10 @@ module Datadog
                 Core::Runtime::Ext::Metrics::METRIC_YJIT_YJIT_ALLOC_SIZE,
                 Core::Environment::YJIT.yjit_alloc_size
               )
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_YJIT_RATIO_IN_YJIT,
+                Core::Environment::YJIT.ratio_in_yjit
+              )
             end
           end
         end

data/lib/datadog/core/semaphore.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    # Semaphore pattern implementation, as described in documentation for
+    # ConditionVariable.
+    #
+    # @api private
+    class Semaphore
+      def initialize
+        @wake_lock = Mutex.new
+        @wake = ConditionVariable.new
+      end
+
+      def signal
+        wake_lock.synchronize do
+          wake.signal
+        end
+      end
+
+      def wait(timeout = nil)
+        wake_lock.synchronize do
+          # steep specifies that the second argument to wait is of type
+          # ::Time::_Timeout which for some reason is not Numeric and is not
+          # castable from Numeric.
+          wake.wait(wake_lock, timeout) # steep:ignore
+        end
+      end
+
+      private
+
+      attr_reader :wake_lock, :wake
+    end
+  end
+end

data/lib/datadog/core/telemetry/component.rb

@@ -5,6 +5,7 @@ require_relative 'event'
 require_relative 'http/transport'
 require_relative 'metrics_manager'
 require_relative 'worker'
+require_relative 'logging'
 
 require_relative '../configuration/ext'
 require_relative '../utils/forking'
@@ -17,6 +18,7 @@ module Datadog
         attr_reader :enabled
 
         include Core::Utils::Forking
+        include Telemetry::Logging
 
         def self.build(settings, agent_settings, logger)
           enabled = settings.telemetry.enabled

data/lib/datadog/core/telemetry/event.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require_relative '../utils/forking'
+require_relative '../utils/sequence'
+
 module Datadog
   module Core
     module Telemetry
@@ -338,7 +341,6 @@ module Datadog
         class Log < Base
           LEVELS = {
             error: 'ERROR',
-            debug: 'DEBUG',
             warn: 'WARN',
           }.freeze
 
@@ -346,19 +348,22 @@ module Datadog
             'logs'
           end
 
-          def initialize(message:, level:)
+          def initialize(message:, level:, stack_trace: nil)
             super()
             @message = message
+            @stack_trace = stack_trace
             @level = LEVELS.fetch(level) { |k| raise ArgumentError, "Invalid log level :#{k}" }
           end
 
           def payload
             {
-              logs: [{
-                message: @message,
-                level: @level,
-                # More optional fields to be added here...
-              }]
+              logs: [
+                {
+                  message: @message,
+                  level: @level,
+                  stack_trace: @stack_trace,
+                }.compact
+              ]
             }
           end
         end

data/lib/datadog/core/telemetry/logger.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Telemetry
+      # === INTERNAL USAGE ONLY ===
+      #
+      # Report telemetry logs via delegating to the telemetry component instance via mutex.
+      #
+      # IMPORTANT: Invoking this method during the lifecycle of component initialization will
+      # be no-op instead.
+      #
+      # For developer using this module:
+      #   read: lib/datadog/core/telemetry/logging.rb
+      module Logger
+        class << self
+          def report(exception, level: :error, description: nil)
+            instance&.report(exception, level: level, description: description)
+          end
+
+          def error(description)
+            instance&.error(description)
+          end
+
+          private
+
+          def instance
+            # Component initialization uses a mutex to avoid having concurrent initialization.
+            # Trying to access the telemetry component during initialization (specifically:
+            # from the thread that's actually doing the initialization) would cause a deadlock,
+            # since accessing the components would try to recursively lock the mutex.
+            #
+            # To work around this, we use allow_initialization: false to avoid triggering this issue.
+            #
+            # The downside is: this leaves us unable to report telemetry during component initialization.
+            components = Datadog.send(:components, allow_initialization: false)
+
+            if components && components.telemetry
+              components.telemetry
+            else
+              Datadog.logger.warn(
+                'Failed to send telemetry before components initialization or within components lifecycle'
+              )
+              nil
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/logging.rb

@@ -2,32 +2,68 @@
 
 require_relative 'event'
 
+require 'pathname'
+
 module Datadog
   module Core
     module Telemetry
-      # Logging interface for sending telemetry logs.
+      # === INTERNAL USAGE ONLY ===
+      #
+      # Logging interface for sending telemetry logs... so we can fix them.
+      #
+      # For developer using this module:
+      # - MUST NOT provide any sensitive information (PII)
+      # - SHOULD reduce the data cardinality for batching/aggregation
+      #
+      # Before using it, ask yourself:
+      # - Do we need to know about this (ie. internal error or client error)?
+      # - How severe/critical is this error? (ie. error, warning, fatal)
+      # - What information needed to make it actionable?
       #
-      # Reporting internal error so that we can fix them.
-      # IMPORTANT: Make sure to not log any sensitive information.
       module Logging
-        module_function
+        # Extract datadog stack trace from the exception
+        module DatadogStackTrace
+          GEM_ROOT = Pathname.new("#{__dir__}/../../../..").cleanpath.to_s
+
+          def self.from(exception)
+            backtrace = exception.backtrace
+
+            return unless backtrace
+            return if backtrace.empty?
+
+            # vendored deps
+            vendored_deps = Gem.path.any? { |p| p.start_with?(GEM_ROOT) }
 
-        def report(exception, level:)
+            backtrace.map do |line|
+              if !vendored_deps && line.start_with?(GEM_ROOT) ||
+                  vendored_deps && line.start_with?(GEM_ROOT) && Gem.path.none? { |p| line.start_with?(p) }
+                line[GEM_ROOT.length..-1] || ''
+              else
+                'REDACTED'
+              end
+            end.join(',')
+          end
+        end
+
+        def report(exception, level: :error, description: nil)
           # Annoymous exceptions to be logged as <Class:0x00007f8b1c0b3b40>
-          message = exception.class.name || exception.class.inspect
+          message = +''
+          message << (exception.class.name || exception.class.inspect)
+          message << ':' << description if description
 
           event = Event::Log.new(
             message: message,
-            level: level
+            level: level,
+            stack_trace: DatadogStackTrace.from(exception)
           )
 
-          if (telemetry = Datadog.send(:components).telemetry)
-            telemetry.log!(event)
-          else
-            Datadog.logger.debug do
-              "Attempting to send telemetry log when telemetry component is not ready: #{message}"
-            end
-          end
+          log!(event)
+        end
+
+        def error(description)
+          event = Event::Log.new(message: description, level: :error)
+
+          log!(event)
         end
       end
     end

data/lib/datadog/core/telemetry/request.rb

@@ -31,6 +31,18 @@ module Datadog
 
           def application
             config = Datadog.configuration
+
+            tracer_version = Core::Environment::Identity.gem_datadog_version_semver2
+
+            # We need some to distinguish datadog-ci gem versions
+            # when examining telemetry metrics emitted from the datadog-ci gem.
+            #
+            # This code checks that Datadog::CI is loaded and ci mode is enabled and adds
+            # "-ci-X.Y.Z" suffix to the tracer version.
+            if defined?(::Datadog::CI::VERSION) && config.respond_to?(:ci) && config.ci.enabled
+              tracer_version = "#{tracer_version}-ci-#{::Datadog::CI::VERSION::STRING}"
+            end
+
             {
               env: config.env,
               language_name: Core::Environment::Ext::LANG,
@@ -39,7 +51,7 @@ module Datadog
               runtime_version: Core::Environment::Ext::ENGINE_VERSION,
               service_name: config.service,
               service_version: config.version,
-              tracer_version: Core::Environment::Identity.gem_datadog_version_semver2
+              tracer_version: tracer_version
             }
           end
 

data/lib/datadog/core/transport/ext.rb

@@ -16,6 +16,7 @@ module Datadog
           #
           # Setting this header to any non-empty value enables this feature.
           HEADER_CLIENT_COMPUTED_TOP_LEVEL = 'Datadog-Client-Computed-Top-Level'
+          HEADER_CLIENT_COMPUTED_STATS = 'Datadog-Client-Computed-Stats'
           HEADER_META_LANG = 'Datadog-Meta-Lang'
           HEADER_META_LANG_VERSION = 'Datadog-Meta-Lang-Version'
           HEADER_META_LANG_INTERPRETER = 'Datadog-Meta-Lang-Interpreter'

data/lib/datadog/core/utils/time.rb

@@ -34,6 +34,18 @@ module Datadog
           define_singleton_method(:now, &block)
         end
 
+        # Overrides the implementation of `#get_time
+        # with the provided callable.
+        #
+        # Overriding the method `#get_time` instead of
+        # indirectly calling `block` removes
+        # one level of method call overhead.
+        #
+        # @param block [Proc] block that accepts unit and returns timestamp in the requested unit
+        def get_time_provider=(block)
+          define_singleton_method(:get_time, &block)
+        end
+
         def measure(unit = :float_second)
           before = get_time(unit)
           yield

data/lib/datadog/core/workers/async.rb

@@ -148,7 +148,7 @@ module Datadog
               end
               # rubocop:enable Lint/RescueException
             end
-            @worker.name = self.class.name unless Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.3')
+            @worker.name = self.class.name
             @worker.thread_variable_set(:fork_safe, true)
 
             nil