datadog 2.22.0 → 2.24.0

data/lib/datadog/core/transport/transport.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require_relative 'http/client'
+
+module Datadog
+  module Core
+    module Transport
+      # Raised when configured with an unknown API version
+      class UnknownApiVersionError < StandardError
+        attr_reader :version
+
+        def initialize(version)
+          super
+
+          @version = version
+        end
+
+        def message
+          "No matching transport API for version #{version}!"
+        end
+      end
+
+      # Raised when the API verson to downgrade to does not map to a
+      # defined API.
+      class NoDowngradeAvailableError < StandardError
+        attr_reader :version
+
+        def initialize(version)
+          super
+
+          @version = version
+        end
+
+        def message
+          "No downgrade from transport API version #{version} is available!"
+        end
+      end
+
+      # Base class for transports.
+      class Transport
+        attr_reader :client, :apis, :default_api, :current_api_id, :logger
+
+        class << self
+          # The HTTP client class to use for requests, derived from
+          # Core::Transport::HTTP::Client.
+          #
+          # Important: this attribute is NOT inherited by derived classes -
+          # it must be set by every Transport class that wants to have a
+          # non-default HTTP::Client instance.
+          attr_accessor :http_client_class
+        end
+
+        def initialize(apis, default_api, logger:)
+          @apis = apis
+          @default_api = default_api
+          @logger = logger
+
+          set_api!(default_api)
+        end
+
+        def current_api
+          apis[current_api_id]
+        end
+
+        private
+
+        def set_api!(api_id)
+          raise UnknownApiVersionError, api_id unless apis.key?(api_id)
+
+          @current_api_id = api_id
+          client_class = self.class.http_client_class || Core::Transport::HTTP::Client
+          @client = client_class.new(current_api, logger: logger) # steep:ignore
+        end
+
+        def downgrade?(response)
+          return false unless apis.fallbacks.key?(current_api_id)
+
+          response.not_found? || response.unsupported?
+        end
+
+        def downgrade!
+          downgrade_api_id = apis.fallbacks[current_api_id]
+          raise NoDowngradeAvailableError, current_api_id if downgrade_api_id.nil?
+
+          set_api!(downgrade_api_id)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/utils/array.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Utils
+      # Common array-related utility functions.
+      module Array
+        def self.filter_map(array, &block)
+          if array.respond_to?(:filter_map)
+            # DEV Supported since Ruby 2.7, saves an intermediate object creation
+            array.filter_map(&block)
+          elsif array.is_a?(Enumerator::Lazy)
+            # You would think that .compact would work here, but it does not:
+            # the result of .map could be an Enumerator::Lazy instance which
+            # does not implement #compact on Ruby 2.5/2.6.
+            array.map(&block).reject do |item|
+              item.nil?
+            end
+          else
+            array.each_with_object([]) do |item, memo|
+              new_item = block.call(item)
+              memo.push(new_item) unless new_item.nil?
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/{appsec/api_security → core/utils}/lru_cache.rb

@@ -3,13 +3,13 @@
 require 'forwardable'
 
 module Datadog
-  module AppSec
-    module APISecurity
+  module Core
+    module Utils
       # An LRU (Least Recently Used) cache implementation that relies on the
       # Ruby 1.9+ `Hash` implementation that guarantees insertion order.
       #
       # WARNING: This implementation is NOT thread-safe and should be used
-      #          in a single-threaded context.
+      #          in a single-threaded context or guarded by Mutex.
       class LRUCache
         extend Forwardable
 
@@ -30,25 +30,14 @@ module Datadog
           end
         end
 
-        def store(key, value)
-          return @store[key] = value if @store.delete(key)
-
-          # NOTE: evict the oldest entry if store reached the maximum allowed size
-          @store.shift if @store.size >= @max_size
-          @store[key] = value
-        end
-
-        # NOTE: If the key exists, it's moved to the end of the list and
-        #       if does not, the given block will be executed and the result
-        #       will be stored (which will add it to the end of the list).
-        def fetch_or_store(key)
-          if (entry = @store.delete(key))
-            return @store[key] = entry
+        def []=(key, value)
+          if @store.delete(key)
+            @store[key] = value
+          else
+            # NOTE: evict the oldest entry if store reached the maximum allowed size
+            @store.shift if @store.size >= @max_size
+            @store[key] = value
           end
-
-          # NOTE: evict the oldest entry if store reached the maximum allowed size
-          @store.shift if @store.size >= @max_size
-          @store[key] = yield
         end
       end
     end

data/lib/datadog/core/utils/network.rb

@@ -21,6 +21,8 @@ module Datadog
           cf-connecting-ipv6
         ].freeze
 
+        CGNAT_IP_RANGE = IPAddr.new('100.64.0.0/10')
+
         class << self
           # Returns a client IP associated with the request if it was
           #   retrieved successfully.
@@ -131,7 +133,7 @@ module Datadog
           end
 
           def global_ip?(parsed_ip)
-            parsed_ip && !parsed_ip.private? && !parsed_ip.loopback? && !parsed_ip.link_local?
+            parsed_ip && !parsed_ip.private? && !parsed_ip.loopback? && !parsed_ip.link_local? && !CGNAT_IP_RANGE.include?(parsed_ip)
           end
         end
       end

data/lib/datadog/core/utils/only_once_successful.rb

@@ -29,7 +29,11 @@ module Datadog
       # In https://github.com/DataDog/dd-trace-rb/pull/1398#issuecomment-797378810 we have a discussion of alternatives,
       # including an alternative implementation that is Ractor-safe once spent.
       class OnlyOnceSuccessful < OnlyOnce
-        def initialize(limit = 0)
+        def initialize(limit = nil)
+          if limit && limit <= 0
+            raise ArgumentError, "Limit must be a positive integer if provided: #{limit}"
+          end
+
           super()
 
           @limit = limit
@@ -63,7 +67,9 @@ module Datadog
 
         private
 
+        # Use this method only after checking that limit is not nil.
         def check_limit!
+          # @type ivar @limit: Integer
           if @retries >= @limit
             @failed = true
             @ran_once = true
@@ -71,7 +77,7 @@ module Datadog
         end
 
         def limited?
-          !@limit.nil? && @limit.positive?
+          !@limit.nil?
         end
 
         def reset_ran_once_state_for_tests

data/lib/datadog/core/utils/time.rb

@@ -10,7 +10,7 @@ module Datadog
         # Current monotonic time
         #
         # @param unit [Symbol] unit for the resulting value, same as ::Process#clock_gettime, defaults to :float_second
-        # @return [Numeric] timestamp in the requested unit, since some unspecified starting point
+        # @return [Float|Integer] timestamp in the requested unit, since some unspecified starting point
         def get_time(unit = :float_second)
           Process.clock_gettime(Process::CLOCK_MONOTONIC, unit)
         end

data/lib/datadog/core/utils.rb

@@ -38,6 +38,8 @@ module Datadog
 
       # Ensure `str` is a valid UTF-8, ready to be
       # sent through the tracer transport.
+      # DEV-3.0: This method should unconditionally handle invalid byte sequences
+      # DEV-3.0: and return a safe string to display.
       #
       # @param [String,#to_s] str object to be converted to a UTF-8 string
       # @param [Boolean] binary whether to expect binary data in the `str` parameter

data/lib/datadog/core/workers/async.rb

@@ -8,6 +8,11 @@ module Datadog
       module Async
         # Adds threading behavior to workers
         # to run tasks asynchronously.
+        #
+        # This module is included in Polling module, and has no other
+        # direct users.
+        #
+        # @api private
         module Thread
           FORK_POLICY_STOP = :stop
           FORK_POLICY_RESTART = :restart
@@ -24,7 +29,11 @@ module Datadog
           # Methods that must be prepended
           module PrependedMethods
             def perform(*args)
-              start_async { self.result = super(*args) } unless started?
+              unless started?
+                start_async do
+                  self.result = super(*args)
+                end
+              end
             end
           end
 

data/lib/datadog/core/workers/interval_loop.rb

@@ -5,6 +5,11 @@ module Datadog
     module Workers
       # Adds looping behavior to workers, with a sleep
       # interval between each loop.
+      #
+      # This module is included in Polling module, and has no other
+      # direct users.
+      #
+      # @api private
       module IntervalLoop
         BACK_OFF_RATIO = 1.2
         BACK_OFF_MAX = 5
@@ -38,15 +43,39 @@ module Datadog
 
         def stop_loop
           mutex.synchronize do
-            return false unless run_loop?
-
+            # Do not call run_loop? from this method to see if the loop
+            # is running, because @run_loop is normally initialized by
+            # the background thread and if the stop is requested right
+            # after the worker starts, the background thread may be created
+            # (and scheduled) but hasn't run yet, thus skipping the
+            # write to @run_loop here would leave the thread running forever.
             @run_loop = false
+
+            # It is possible that we don't need to signal shutdown if
+            # @run_loop was not initialized (i.e. we changed it from not
+            # defined to false above). But let's be safe and signal the
+            # shutdown anyway, I don't see what harm it can cause.
             shutdown.signal
           end
 
+          # Previously, this method would return false (and do nothing)
+          # if the worker was not running the loop. However, this was racy -
+          # see https://github.com/DataDog/ruby-guild/issues/279.
+          # stop_loop now always sets the state to "stop requested" and,
+          # correspondingly, always returns true.
+          #
+          # There is some test code that returns false when mocking this
+          # method - most likely this method should be treated as a void one
+          # and the caller should assume that the stop was always requested.
           true
         end
 
+        # TODO This overwrites Queue's +work_pending?+ method with an
+        # implementation that, to me, is at leat questionable semantically:
+        # the Queue's idea of pending work is if the buffer is not empty,
+        # but this module says that work is pending if the work processing
+        # loop is scheduled to run (in other words, as long as the background
+        # thread is running, there is always pending work).
         def work_pending?
           run_loop?
         end
@@ -104,7 +133,19 @@ module Datadog
 
         def perform_loop
           mutex.synchronize do
-            @run_loop = true
+            unless defined?(@run_loop)
+              # This write must only happen if @run_loop is not defined
+              # (i.e., not initialized). In the case when the worker is
+              # asked to stop right after it is created, the thread may not
+              # have run yet by the time +stop_loop+ is invoked and
+              # we need to preserve the stop-requested state from
+              # +stop_loop+ to +perform_loop+.
+              #
+              # If the workers are refactored to use classes and inheritance
+              # and their state, such as @run_loop, is initialized in
+              # constructors, the write can be made unconditional.
+              @run_loop = true
+            end
 
             shutdown.wait(mutex, loop_wait_time) if loop_wait_before_first_iteration?
           end

data/lib/datadog/core/workers/polling.rb

@@ -7,6 +7,8 @@ module Datadog
   module Core
     module Workers
       # Adds polling (async looping) behavior to workers
+      #
+      # @api private
       module Polling
         DEFAULT_SHUTDOWN_TIMEOUT = 1
 

data/lib/datadog/core/workers/queue.rb

@@ -5,6 +5,17 @@ module Datadog
     module Workers
       # Adds queue behavior to workers, with a buffer
       # to which items can be queued then dequeued.
+      #
+      # This module is included in some but not all workers.
+      # Notably, Data Streams Processor uses a queue but implements it
+      # inline rather than using this module.
+      #
+      # The workers that do include Queue also include Polling, which
+      # in turn includes Async::Thread and IntervalLoop. This means
+      # we have e.g. +in_iteration?+ always available in any worker
+      # that includes Queue.
+      #
+      # @api private
       module Queue
         def self.included(base)
           base.prepend(PrependedMethods)
@@ -13,11 +24,16 @@ module Datadog
         # Methods that must be prepended
         module PrependedMethods
           def perform(*args)
-            super(*dequeue) if work_pending?
+            if work_pending?
+              work = dequeue
+              super(*work)
+            end
           end
         end
 
         def buffer
+          # Why is this an unsynchronized Array and not a Core::Buffer
+          # instance?
           @buffer ||= []
         end
 
@@ -34,10 +50,93 @@ module Datadog
           !buffer.empty?
         end
 
+        # Wait for the worker to finish handling all work that has already
+        # been submitted to it.
+        #
+        # If the worker is not enabled, returns nil.
+        # If the worker is enabled, returns whether, at the point of return,
+        # there was no pending or in progress work.
+        #
+        # Flushing can time out because there is a constant stream of work
+        # submitted at the same or higher rate than it is processed.
+        # Flushing can also fail if the worker thread is not running -
+        # this method will not flush from the calling thread.
+        def flush(timeout: nil)
+          # Default timeout is 5 seconds.
+          # Specific workers can override it to be more or less
+          timeout ||= 5
+
+          # Nothing needs to be done if the worker is not enabled.
+          return nil unless enabled?
+
+          unless running?
+            unless buffer.empty?
+              # If we are asked to flush but the worker is not running,
+              # do not flush from the caller thread. If the buffer is not
+              # empty, it will not be flushed. Log a warning to this effect.
+              #
+              # We are not guaranteed to have a logger as an instance method,
+              # reference the global for now - all other worker methods
+              # also reference the logger globally.
+              # TODO inject it into worker instances.
+              Datadog.logger.debug { "Asked to flush #{self} when the worker is not running" }
+              return false
+            end
+          end
+
+          started = Utils::Time.get_time
+          loop do
+            # The AppStarted event is triggered by the worker itself,
+            # from the worker thread. As such the main thread has no way
+            # to delay itself until that event is queued and we need some
+            # way to wait until that event is sent out to assert on it in
+            # the test suite. Check the run once flag which *should*
+            # indicate the event has been queued (at which point our queue
+            # depth check should wait until it's sent).
+            # This is still a hack because the flag can be overridden
+            # either way with or without the event being sent out.
+            # Note that if the AppStarted sending fails, this check
+            # will return false and flushing will be blocked until the
+            # 15 second timeout.
+            # Note that the first wait interval between telemetry event
+            # sending is 10 seconds, the timeout needs to be strictly
+            # greater than that.
+            return true if idle?
+
+            return false if Utils::Time.get_time - started > timeout
+
+            sleep 0.5
+          end
+        end
+
         protected
 
         attr_writer \
           :buffer
+
+        # Returns whether this worker has no pending work and is not actively
+        # working.
+        #
+        # The reason why "actively working" is considered is that we use
+        # flushing to ensure all work is completed before asserting on the
+        # outcome in the tests - if work is happening in a background thread,
+        # it's too early to assert on its results.
+        def idle?
+          # We have a +work_pending?+ method in this class that semantically
+          # would be appropriate here instead of calling +buffer.empty?+.
+          # Unfortunately IntervalLoop replaces our implementation of
+          # +work_pending?+ with one that doesn't make sense at least for the
+          # Queue. And we can't change the order of module includes because
+          # they all override +perform+ and the correct behavior depends on
+          # placing IntervalLoop after Queue.
+          #
+          # The TraceWriter worker then defines +work_pending?+ to be the
+          # same as Queue implementation here... Essentially, it demands
+          # the behavior that perhaps should be applied to all workers.
+          #
+          # Until this mess is untangled, call +buffer.empty?+ here.
+          buffer.empty? && !in_iteration?
+        end
       end
     end
   end

data/lib/datadog/data_streams/configuration/settings.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require_relative '../../core/environment/variable_helpers'
+require_relative '../ext'
+
+module Datadog
+  module DataStreams
+    module Configuration
+      # Configuration settings for Data Streams Monitoring.
+      module Settings
+        def self.extended(base)
+          base = base.singleton_class unless base.is_a?(Class)
+          add_settings!(base)
+        end
+
+        def self.add_settings!(base)
+          base.class_eval do
+            # Data Streams Monitoring configuration
+            # @public_api
+            settings :data_streams do
+              # Whether Data Streams Monitoring is enabled. When enabled, the library will
+              # collect and report data lineage information for messaging systems.
+              #
+              # @default `DD_DATA_STREAMS_ENABLED` environment variable, otherwise `false`.
+              # @return [Boolean]
+              option :enabled do |o|
+                o.type :bool
+                o.env Ext::ENV_ENABLED
+                o.default false
+              end
+
+              # The interval (in seconds) at which Data Streams Monitoring stats are flushed.
+              #
+              # @default 10.0
+              # @env '_DD_TRACE_STATS_WRITER_INTERVAL'
+              # @return [Float]
+              # @!visibility private
+              option :interval do |o|
+                o.type :float
+                o.env '_DD_TRACE_STATS_WRITER_INTERVAL'
+                o.default 10.0
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/data_streams/configuration.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require_relative 'configuration/settings'
+
+module Datadog
+  module DataStreams
+    # Configuration for Data Streams Monitoring
+    module Configuration
+    end
+  end
+end

data/lib/datadog/data_streams/ext.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DataStreams
+    # Constants for Data Streams Monitoring configuration
+    # @public_api
+    module Ext
+      ENV_ENABLED = 'DD_DATA_STREAMS_ENABLED'
+    end
+  end
+end

data/lib/datadog/data_streams/extensions.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require_relative 'configuration'
+require_relative '../core/configuration'
+
+module Datadog
+  module DataStreams
+    # Extends Datadog with Data Streams Monitoring features
+    module Extensions
+      # Inject Data Streams settings into global configuration.
+      def self.activate!
+        Core::Configuration::Settings.extend(Configuration::Settings)
+      end
+    end
+  end
+end