datadog 2.4.0 → 2.44.0

data/lib/datadog/core/utils/only_once_successful.rb

@@ -1,24 +1,39 @@
 # frozen_string_literal: true
 
-require_relative 'only_once'
+require_relative "only_once"
 
 module Datadog
   module Core
     module Utils
-      # Helper class to execute something with only one success.
+      # Helper class to execute something with only one successful execution.
       #
-      # This is useful for cases where we want to ensure that a block of code is only executed once, and only if it
-      # succeeds. One such example is sending app-started telemetry event.
+      # If limit is not provided to the constructor, +run+ will execute the
+      # block an unlimited number of times until the block indicates that it
+      # executed successfully by returning a truthy value. After a block
+      # executes successfully, subsequent +run+ calls will not invoke the
+      # block.
       #
-      # Successful execution is determined by the return value of the block: any truthy value is considered success.
+      # If a non-zero limit is provided to the constructor, +run+ will
+      # execute the block up to that many times, and will mark the instance
+      # of OnlyOneSuccessful as failed if none of the executions succeeded.
       #
-      # Thread-safe when used correctly (e.g. be careful of races when lazily initializing instances of this class).
+      # One consumer of this class is sending the app-started telemetry event.
+      #
+      # Successful execution is determined by the return value of the block:
+      # any truthy value is considered success.
+      #
+      # This class is thread-safe (however, instances of it must also be
+      # created in a thread-safe manner).
       #
       # Note: In its current state, this class is not Ractor-safe.
       # In https://github.com/DataDog/dd-trace-rb/pull/1398#issuecomment-797378810 we have a discussion of alternatives,
       # including an alternative implementation that is Ractor-safe once spent.
       class OnlyOnceSuccessful < OnlyOnce
-        def initialize(limit = 0)
+        def initialize(limit = nil)
+          if limit && limit <= 0
+            raise ArgumentError, "Limit must be a positive integer if provided: #{limit}"
+          end
+
           super()
 
           @limit = limit
@@ -52,7 +67,9 @@ module Datadog
 
         private
 
+        # Use this method only after checking that limit is not nil.
         def check_limit!
+          # @type ivar @limit: Integer
           if @retries >= @limit
             @failed = true
             @ran_once = true
@@ -60,7 +77,7 @@ module Datadog
         end
 
         def limited?
-          !@limit.nil? && @limit.positive?
+          !@limit.nil?
         end
 
         def reset_ran_once_state_for_tests

data/lib/datadog/core/utils/safe_dup.rb

@@ -5,8 +5,8 @@ module Datadog
     module Utils
       # Helper methods for safer dup
       module SafeDup
-        # String#+@ was introduced in Ruby 2.3
-        def self.frozen_or_dup(v)
+        # Steep: https://github.com/soutaro/steep/issues/2001
+        def self.frozen_or_dup(v) # steep:ignore MethodBodyTypeMismatch
           # For the case of a String we use the methods +@ and -@.
           # Those methods are only for String objects
           # they are faster and chepaer on the memory side.
@@ -22,7 +22,8 @@ module Datadog
           end
         end
 
-        def self.frozen_dup(v)
+        # Steep: https://github.com/soutaro/steep/issues/2001
+        def self.frozen_dup(v) # steep:ignore MethodBodyTypeMismatch
           # For the case of a String we use the methods -@
           # That method are only for String objects
           # they are faster and chepaer on the memory side.

data/lib/datadog/core/utils/sequence.rb

@@ -5,16 +5,15 @@ module Datadog
     module Utils
       # Generates values from a consistent sequence
       class Sequence
-        def initialize(seed = 0, &block)
+        def initialize(seed = 0)
           @seed = seed
           @current = seed
-          @next_item = block
         end
 
         def next
-          next_item = @next_item ? @next_item.call(@current) : @current
+          current = @current
           @current += 1
-          next_item
+          current
         end
 
         def reset!

data/lib/datadog/core/utils/spawn_monkey_patch.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require_relative "../../ruby_version"
+
+module Datadog
+  module Core
+    module Utils
+      # Applies the Process.spawn wrapper used to merge additional environment variables
+      # into child processes.
+      module SpawnMonkeyPatch
+        # @param env_provider [#call] returns a Hash of env vars to merge into the child process
+        def self.apply!(env_provider:)
+          @env_provider = env_provider
+
+          # Idempotent: tests, reloads, or repeated Components init must not stack prepends.
+          return if ::Process.singleton_class.ancestors.include?(ProcessSpawnPatch)
+
+          ::Process.singleton_class.prepend(ProcessSpawnPatch)
+        end
+
+        # Prepends `Process.spawn` to merge `env_provider` output into the child's environment hash.
+        module ProcessSpawnPatch
+          # The One and Only Correct Delegation Pattern
+          if RubyVersion.is?(">= 3")
+            def spawn(*args, **kwargs) # steep:ignore DifferentMethodParameterKind
+              super(*SpawnMonkeyPatch.inject_envs(args), **kwargs)
+            end
+          else
+            def spawn(*args)
+              super(*SpawnMonkeyPatch.inject_envs(args))
+            end
+            ruby2_keywords :spawn if respond_to?(:ruby2_keywords, true)
+          end
+        end
+
+        # Merge the env vars from `env_provider` with the optional env `Hash` from {Process.spawn}.
+        #
+        # `env` is the first argument when it is a {Hash}; see MRI `spawn([env, ] *args, options)`:
+        # https://docs.ruby-lang.org/en/master/Process.html#method-c-spawn
+        #
+        # When there is **no** leading env Hash, MRI inherits the parent's `ENV`; we prepend only the
+        # `env_provider` hash so spawned children see parent env plus injections.
+        #
+        # When callers pass `unsetenv_others: true`, MRI only forwards the explicitly passed env Hash;
+        # replacing a missing hash with DATADOG_ENV.to_h would wrongly carry over parent variables.
+        # Prepending only the provider hash preserves `unsetenv_others` semantics.
+        #
+        # See https://docs.ruby-lang.org/en/master/Process.html#module-Process-label-Environment+Variables+-28-3Aunsetenv_others-29
+        #
+        # NOTE: `::Hash` (not bare `Hash`) is required because this module is nested under
+        # `Datadog::Core::Utils`, and `Datadog::Core::Utils::Hash` exists.
+        # Bare `Hash` resolves to that module via Module.nesting, making `Hash === some_hash`
+        # silently return `false`. See https://github.com/DataDog/dd-trace-rb/issues/5621.
+        def self.inject_envs(args)
+          provided_env = @env_provider.call
+
+          if ::Hash === args.first
+            args[0] = args.first.merge(provided_env)
+          else
+            args.unshift(provided_env)
+          end
+
+          args
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/utils/time.rb

@@ -5,55 +5,41 @@ module Datadog
     module Utils
       # Common database-related utility functions.
       module Time
-        module_function
+        # timecop monkey-patches Time.now, Process.clock_gettime, etc:
+        # https://github.com/travisjeffery/timecop/blob/v0.9.11/lib/timecop/time_extensions.rb
+        # It keeps the original methods as aliases.
+        # We want the real not-mocked time here, so we use those aliases to the original method if present.
+        # If not present we assume these methods have not been monkey-patched and are the original.
+
+        original_clock_gettime_name = Process.respond_to?(:clock_gettime_without_mock) ? :clock_gettime_without_mock : :clock_gettime
+        define_singleton_method(:original_clock_gettime, &Process.method(original_clock_gettime_name))
+
+        MONOTONIC_CLOCK_ID = RUBY_PLATFORM.include?("darwin") ? Process::CLOCK_MONOTONIC_RAW : Process::CLOCK_MONOTONIC
 
         # Current monotonic time
+        # On macOS, CLOCK_MONOTONIC only has microsecond precision,
+        # so we use CLOCK_MONOTONIC_RAW which has nanosecond precision instead.
         #
         # @param unit [Symbol] unit for the resulting value, same as ::Process#clock_gettime, defaults to :float_second
-        # @return [Numeric] timestamp in the requested unit, since some unspecified starting point
-        def get_time(unit = :float_second)
-          Process.clock_gettime(Process::CLOCK_MONOTONIC, unit)
+        # @return [Float|Integer] timestamp in the requested unit, since some unspecified starting point
+        def self.get_time(unit = :float_second)
+          original_clock_gettime(MONOTONIC_CLOCK_ID, unit)
         end
 
         # Current wall time.
         #
         # @return [Time] current time object
-        def now
-          ::Time.now
-        end
-
-        # Overrides the implementation of `#now
-        # with the provided callable.
-        #
-        # Overriding the method `#now` instead of
-        # indirectly calling `block` removes
-        # one level of method call overhead.
-        #
-        # @param block [Proc] block that returns a `Time` object representing the current wall time
-        def now_provider=(block)
-          define_singleton_method(:now, &block)
-        end
-
-        # Overrides the implementation of `#get_time
-        # with the provided callable.
-        #
-        # Overriding the method `#get_time` instead of
-        # indirectly calling `block` removes
-        # one level of method call overhead.
-        #
-        # @param block [Proc] block that accepts unit and returns timestamp in the requested unit
-        def get_time_provider=(block)
-          define_singleton_method(:get_time, &block)
-        end
+        original_time_now_name = ::Time.respond_to?(:now_without_mock_time) ? :now_without_mock_time : :now
+        define_singleton_method(:now, &::Time.method(original_time_now_name))
 
-        def measure(unit = :float_second)
+        def self.measure(unit = :float_second)
           before = get_time(unit)
           yield
           after = get_time(unit)
           after - before
         end
 
-        def as_utc_epoch_ns(time)
+        def self.as_utc_epoch_ns(time)
           # we use #to_r instead of #to_f because Float doesn't have enough precision to represent exact nanoseconds, see
           # https://rubyapi.org/3.0/o/time#method-i-to_f
           (time.to_r * 1_000_000_000).to_i

data/lib/datadog/core/utils/truncation.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Utils
+      # Helper methods for truncating data
+      module Truncation
+        module_function
+
+        def truncate_in_middle(string, max_prefix_length, max_suffix_length)
+          max_length = max_prefix_length + 3 + max_suffix_length
+          if string.length > max_length
+            "#{string[0...max_prefix_length]}...#{string[-max_suffix_length..-1]}"
+          else
+            string
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/utils/url.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'uri'
+require "uri"
 
 module Datadog
   module Core

data/lib/datadog/core/utils.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative 'utils/forking'
+require_relative "utils/forking"
 
 module Datadog
   module Core
@@ -9,7 +9,7 @@ module Datadog
     module Utils
       extend Forking
 
-      EMPTY_STRING = ''.encode(::Encoding::UTF_8).freeze
+      EMPTY_STRING = "".encode(::Encoding::UTF_8).freeze
 
       # Stringifies `value` and ensures the outcome is
       # string is no longer than `size`.
@@ -20,7 +20,7 @@ module Datadog
       # If `omission.size` is larger than `size`, the output
       # will still be `omission.size` in length.
       # @!visibility private
-      def self.truncate(value, size, omission = '...')
+      def self.truncate(value, size, omission = "...")
         string = value.to_s
 
         return string if string.size <= size
@@ -38,6 +38,8 @@ module Datadog
 
       # Ensure `str` is a valid UTF-8, ready to be
       # sent through the tracer transport.
+      # DEV-3.0: This method should unconditionally handle invalid byte sequences
+      # DEV-3.0: and return a safe string to display.
       #
       # @param [String,#to_s] str object to be converted to a UTF-8 string
       # @param [Boolean] binary whether to expect binary data in the `str` parameter
@@ -50,7 +52,7 @@ module Datadog
         if binary
           # This option is useful for "gracefully" displaying binary data that
           # often contains text such as marshalled objects
-          str.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '')
+          str.encode("UTF-8", "binary", invalid: :replace, undef: :replace, replace: "")
         elsif str.encoding == ::Encoding::UTF_8
           str
         elsif str.empty?
@@ -61,11 +63,18 @@ module Datadog
           str.encode(::Encoding::UTF_8)
         end
       rescue => e
-        Datadog.logger.debug("Error encoding string in UTF-8: #{e}")
+        Datadog.logger.debug("Error encoding string in UTF-8: #{e.class}: #{e.message}")
 
         placeholder
       end
 
+      def self.encode_tags(hash)
+        # Make sure everything is an utf-8 string, to avoid encoding issues in downstream
+        hash.each_with_object({}) do |(key, value), h|
+          h[utf8_encode(key)] = utf8_encode(value)
+        end
+      end
+
       # @!visibility private
       def self.without_warnings
         # This is typically used when monkey patching functions such as

data/lib/datadog/core/vendor/multipart-post/multipart/post/composite_read_io.rb

@@ -108,7 +108,7 @@ module Datadog
             end
 
             def respond_to?(meth, include_all = false)
-              @io.respond_to?(meth, include_all) || super(meth, include_all)
+              @io.respond_to?(meth, include_all) || super
             end
           end
         end

data/lib/datadog/core/vendor/multipart-post/multipart/post/multipartable.rb

@@ -28,29 +28,29 @@ module Datadog
               "--#{SecureRandom.uuid}"
             end
 
-            def initialize(path, params, headers={}, boundary = Multipartable.secure_boundary)
+            def initialize(path, params, headers = {}, boundary = Multipartable.secure_boundary)
               headers = headers.clone # don't want to modify the original variable
               parts_headers = headers.delete(:parts) || {}
               super(path, headers)
-              parts = params.map do |k,v|
+              parts = params.map do |k, v|
                 case v
                 when Array
-                  v.map {|item| Parts::Part.new(boundary, "#{k}[]", item, parts_headers[k]) }
+                  v.map { |item| Parts::Part.new(boundary, "#{k}[]", item, parts_headers[k]) }
                 else
                   Parts::Part.new(boundary, k, v, parts_headers[k])
                 end
               end.flatten
               parts << Parts::EpiloguePart.new(boundary)
-              ios = parts.map {|p| p.to_io }
-              self.set_content_type(headers["Content-Type"] || "multipart/form-data",
-                                    { "boundary" => boundary })
-              self.content_length = parts.inject(0) {|sum,i| sum + i.length }
+              ios = parts.map { |p| p.to_io }
+              set_content_type(headers["Content-Type"] || "multipart/form-data",
+                {"boundary" => boundary})
+              self.content_length = parts.inject(0) { |sum, i| sum + i.length }
               self.body_stream = CompositeReadIO.new(*ios)
 
               @boundary = boundary
             end
 
-            attr :boundary
+            attr_reader :boundary
           end
         end
       end

data/lib/datadog/core/vendor/multipart-post/multipart/post/parts.rb

@@ -58,7 +58,7 @@ module Datadog
                 part = ''
                 part << "--#{boundary}\r\n"
                 part << "Content-ID: #{headers["Content-ID"]}\r\n" if headers["Content-ID"]
-                part << "Content-Disposition: form-data; name=\"#{name.to_s}\"\r\n"
+                part << "Content-Disposition: form-data; name=\"#{name}\"\r\n"
                 part << "Content-Type: #{headers["Content-Type"]}\r\n" if headers["Content-Type"]
                 part << "\r\n"
                 part << "#{value}\r\n"
@@ -76,9 +76,9 @@ module Datadog
               # @param io [IO]
               # @param headers [Hash]
               def initialize(boundary, name, io, headers = {})
-                file_length = io.respond_to?(:length) ?  io.length : File.size(io.local_path)
+                file_length = io.respond_to?(:length) ? io.length : File.size(io.local_path)
                 @head = build_head(boundary, name, io.original_filename, io.content_type, file_length,
-                                  io.respond_to?(:opts) ? io.opts.merge(headers) : headers)
+                  io.respond_to?(:opts) ? io.opts.merge(headers) : headers)
                 @foot = "\r\n"
                 @length = @head.bytesize + file_length + @foot.length
                 @io = CompositeReadIO.new(StringIO.new(@head), io, StringIO.new(@foot))
@@ -98,16 +98,16 @@ module Datadog
 
                 part = ''
                 part << "--#{boundary}\r\n"
-                part << "Content-Disposition: #{content_disposition}; name=\"#{name.to_s}\"; filename=\"#{filename}\"\r\n"
+                part << "Content-Disposition: #{content_disposition}; name=\"#{name}\"; filename=\"#{filename}\"\r\n"
                 part << "Content-Length: #{content_len}\r\n"
                 if content_id = opts.delete("Content-ID")
                   part << "Content-ID: #{content_id}\r\n"
                 end
 
-                if opts["Content-Type"] != nil
-                  part <<  "Content-Type: " + opts["Content-Type"] + "\r\n"
+                part << if !opts["Content-Type"].nil?
+                  "Content-Type: " + opts["Content-Type"] + "\r\n"
                 else
-                  part << "Content-Type: #{type}\r\n"
+                  "Content-Type: #{type}\r\n"
                 end
 
                 part << "Content-Transfer-Encoding: #{trans_encoding}\r\n"

data/lib/datadog/core/worker.rb

@@ -12,7 +12,7 @@ module Datadog
       end
 
       def perform(*args)
-        task.call(*args) unless task.nil?
+        task&.call(*args)
       end
 
       protected

data/lib/datadog/core/workers/async.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative '../logger'
+require_relative "../logger"
 
 module Datadog
   module Core
@@ -8,6 +8,11 @@ module Datadog
       module Async
         # Adds threading behavior to workers
         # to run tasks asynchronously.
+        #
+        # This module is included in Polling module, and has no other
+        # direct users.
+        #
+        # @api private
         module Thread
           FORK_POLICY_STOP = :stop
           FORK_POLICY_RESTART = :restart
@@ -24,7 +29,11 @@ module Datadog
           # Methods that must be prepended
           module PrependedMethods
             def perform(*args)
-              start_async { self.result = super(*args) } unless started?
+              unless started?
+                start_async do
+                  self.result = super(*args)
+                end
+              end
             end
           end
 
@@ -47,6 +56,14 @@ module Datadog
             @run_async = false
             Datadog.logger.debug { "Forcibly terminating worker thread for: #{self}" }
             worker.terminate
+            # Wait for the worker thread to end
+            begin
+              Timeout.timeout(SHUTDOWN_TIMEOUT) do
+                worker.join
+              end
+            rescue Timeout::Error
+              Datadog.logger.debug { "Worker thread did not end after #{SHUTDOWN_TIMEOUT} seconds: #{self}" }
+            end
             true
           end
 
@@ -112,23 +129,33 @@ module Datadog
             @worker ||= nil
           end
 
+          # Returns true if worker thread is successfully started,
+          # false if it is not started. Reasons for not starting the worker
+          # thread: it is already running, or the process forked and fork
+          # policy is to stop the workers on fork (which means they are
+          # not started in children, really).
           def start_async(&block)
             mutex.synchronize do
-              return if running?
+              return false if running?
 
               if forked?
                 case fork_policy
                 when FORK_POLICY_STOP
                   stop_fork
+                  false
                 when FORK_POLICY_RESTART
+                  # restart_after_fork should return true
                   restart_after_fork(&block)
                 end
               elsif !run_async?
+                # start_worker should return true
                 start_worker(&block)
               end
             end
           end
 
+          # Returns true if worker thread is successfully started,
+          # which should generally be always.
           def start_worker
             @run_async = true
             @pid = Process.pid
@@ -136,22 +163,21 @@ module Datadog
             Datadog.logger.debug { "Starting thread for: #{self}" }
 
             @worker = ::Thread.new do
-              begin
-                yield
-              # rubocop:disable Lint/RescueException
-              rescue Exception => e
-                @error = e
-                Datadog.logger.debug(
-                  "Worker thread error. Cause: #{e.class.name} #{e.message} Location: #{Array(e.backtrace).first}"
-                )
-                raise
-              end
+              yield
+            # rubocop:disable Lint/RescueException
+            rescue Exception => e
+              @error = e
+              Datadog.logger.debug(
+                "Worker thread error. Cause: #{e.class}: #{e.message} Location: #{Array(e.backtrace).first}"
+              )
+              raise
+
               # rubocop:enable Lint/RescueException
             end
-            @worker.name = self.class.name unless Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.3')
+            @worker.name = self.class.name
             @worker.thread_variable_set(:fork_safe, true)
 
-            nil
+            true
           end
 
           def stop_fork

data/lib/datadog/core/workers/interval_loop.rb

@@ -5,6 +5,11 @@ module Datadog
     module Workers
       # Adds looping behavior to workers, with a sleep
       # interval between each loop.
+      #
+      # This module is included in Polling module, and has no other
+      # direct users.
+      #
+      # @api private
       module IntervalLoop
         BACK_OFF_RATIO = 1.2
         BACK_OFF_MAX = 5
@@ -21,25 +26,67 @@ module Datadog
         # Methods that must be prepended
         module PrependedMethods
           def perform(*args)
-            perform_loop { super(*args) }
+            perform_loop do
+              @in_iteration = true
+              begin
+                super(*args)
+              ensure
+                @in_iteration = false
+              end
+            end
+          end
+
+          def in_iteration?
+            defined?(@in_iteration) && @in_iteration
           end
         end
 
         def stop_loop
           mutex.synchronize do
-            return false unless run_loop?
-
+            # Do not call run_loop? from this method to see if the loop
+            # is running, because @run_loop is normally initialized by
+            # the background thread and if the stop is requested right
+            # after the worker starts, the background thread may be created
+            # (and scheduled) but hasn't run yet, thus skipping the
+            # write to @run_loop here would leave the thread running forever.
             @run_loop = false
+
+            # It is possible that we don't need to signal shutdown if
+            # @run_loop was not initialized (i.e. we changed it from not
+            # defined to false above). But let's be safe and signal the
+            # shutdown anyway, I don't see what harm it can cause.
             shutdown.signal
           end
 
+          # Previously, this method would return false (and do nothing)
+          # if the worker was not running the loop. However, this was racy -
+          # see https://github.com/DataDog/ruby-guild/issues/279.
+          # stop_loop now always sets the state to "stop requested" and,
+          # correspondingly, always returns true.
+          #
+          # There is some test code that returns false when mocking this
+          # method - most likely this method should be treated as a void one
+          # and the caller should assume that the stop was always requested.
           true
         end
 
+        # TODO: This method carries two semantics today:
+        # 1. Is there work available to process?
+        # 2. Should the main loop keep running?
+        # Things get messy when `Queue` is mixed in,
+        # wanting to override semantic 1 but not 2.
+        # This should probably be split into two methods:
+        # `work_pending?` (semantic 1) and `continue_loop?` (semantic 2).
+        # Today, `run_loop?` performs semantic 2, but should probably be
+        # renamed to `continue_loop?`, since that would make it clear that
+        # we're just checking if we can "keep going inside the loop",
+        # and not checking if we should run the loop right now.
+        # Clean up {Workers::Queue} after this.
         def work_pending?
           run_loop?
         end
 
+        # TODO: Probably should be renamed to `continue_loop?`, see `work_pending?` TODO.
         def run_loop?
           return false unless instance_variable_defined?(:@run_loop)
 
@@ -93,7 +140,19 @@ module Datadog
 
         def perform_loop
           mutex.synchronize do
-            @run_loop = true
+            unless defined?(@run_loop)
+              # This write must only happen if @run_loop is not defined
+              # (i.e., not initialized). In the case when the worker is
+              # asked to stop right after it is created, the thread may not
+              # have run yet by the time +stop_loop+ is invoked and
+              # we need to preserve the stop-requested state from
+              # +stop_loop+ to +perform_loop+.
+              #
+              # If the workers are refactored to use classes and inheritance
+              # and their state, such as @run_loop, is initialized in
+              # constructors, the write can be made unconditional.
+              @run_loop = true
+            end
 
             shutdown.wait(mutex, loop_wait_time) if loop_wait_before_first_iteration?
           end