datadog 2.8.0 → 2.39.0

data/lib/datadog/core/utils.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative 'utils/forking'
+require_relative "utils/forking"
 
 module Datadog
   module Core
@@ -9,7 +9,7 @@ module Datadog
     module Utils
       extend Forking
 
-      EMPTY_STRING = ''.encode(::Encoding::UTF_8).freeze
+      EMPTY_STRING = "".encode(::Encoding::UTF_8).freeze
 
       # Stringifies `value` and ensures the outcome is
       # string is no longer than `size`.
@@ -20,7 +20,7 @@ module Datadog
       # If `omission.size` is larger than `size`, the output
       # will still be `omission.size` in length.
       # @!visibility private
-      def self.truncate(value, size, omission = '...')
+      def self.truncate(value, size, omission = "...")
         string = value.to_s
 
         return string if string.size <= size
@@ -38,6 +38,8 @@ module Datadog
 
       # Ensure `str` is a valid UTF-8, ready to be
       # sent through the tracer transport.
+      # DEV-3.0: This method should unconditionally handle invalid byte sequences
+      # DEV-3.0: and return a safe string to display.
       #
       # @param [String,#to_s] str object to be converted to a UTF-8 string
       # @param [Boolean] binary whether to expect binary data in the `str` parameter
@@ -50,7 +52,7 @@ module Datadog
         if binary
           # This option is useful for "gracefully" displaying binary data that
           # often contains text such as marshalled objects
-          str.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '')
+          str.encode("UTF-8", "binary", invalid: :replace, undef: :replace, replace: "")
         elsif str.encoding == ::Encoding::UTF_8
           str
         elsif str.empty?
@@ -61,11 +63,18 @@ module Datadog
           str.encode(::Encoding::UTF_8)
         end
       rescue => e
-        Datadog.logger.debug("Error encoding string in UTF-8: #{e}")
+        Datadog.logger.debug("Error encoding string in UTF-8: #{e.class}: #{e.message}")
 
         placeholder
       end
 
+      def self.encode_tags(hash)
+        # Make sure everything is an utf-8 string, to avoid encoding issues in downstream
+        hash.each_with_object({}) do |(key, value), h|
+          h[utf8_encode(key)] = utf8_encode(value)
+        end
+      end
+
       # @!visibility private
       def self.without_warnings
         # This is typically used when monkey patching functions such as

data/lib/datadog/core/vendor/multipart-post/multipart/post/composite_read_io.rb

@@ -108,7 +108,7 @@ module Datadog
             end
 
             def respond_to?(meth, include_all = false)
-              @io.respond_to?(meth, include_all) || super(meth, include_all)
+              @io.respond_to?(meth, include_all) || super
             end
           end
         end

data/lib/datadog/core/vendor/multipart-post/multipart/post/multipartable.rb

@@ -28,29 +28,29 @@ module Datadog
               "--#{SecureRandom.uuid}"
             end
 
-            def initialize(path, params, headers={}, boundary = Multipartable.secure_boundary)
+            def initialize(path, params, headers = {}, boundary = Multipartable.secure_boundary)
               headers = headers.clone # don't want to modify the original variable
               parts_headers = headers.delete(:parts) || {}
               super(path, headers)
-              parts = params.map do |k,v|
+              parts = params.map do |k, v|
                 case v
                 when Array
-                  v.map {|item| Parts::Part.new(boundary, "#{k}[]", item, parts_headers[k]) }
+                  v.map { |item| Parts::Part.new(boundary, "#{k}[]", item, parts_headers[k]) }
                 else
                   Parts::Part.new(boundary, k, v, parts_headers[k])
                 end
               end.flatten
               parts << Parts::EpiloguePart.new(boundary)
-              ios = parts.map {|p| p.to_io }
-              self.set_content_type(headers["Content-Type"] || "multipart/form-data",
-                                    { "boundary" => boundary })
-              self.content_length = parts.inject(0) {|sum,i| sum + i.length }
+              ios = parts.map { |p| p.to_io }
+              set_content_type(headers["Content-Type"] || "multipart/form-data",
+                {"boundary" => boundary})
+              self.content_length = parts.inject(0) { |sum, i| sum + i.length }
               self.body_stream = CompositeReadIO.new(*ios)
 
               @boundary = boundary
             end
 
-            attr :boundary
+            attr_reader :boundary
           end
         end
       end

data/lib/datadog/core/vendor/multipart-post/multipart/post/parts.rb

@@ -58,7 +58,7 @@ module Datadog
                 part = ''
                 part << "--#{boundary}\r\n"
                 part << "Content-ID: #{headers["Content-ID"]}\r\n" if headers["Content-ID"]
-                part << "Content-Disposition: form-data; name=\"#{name.to_s}\"\r\n"
+                part << "Content-Disposition: form-data; name=\"#{name}\"\r\n"
                 part << "Content-Type: #{headers["Content-Type"]}\r\n" if headers["Content-Type"]
                 part << "\r\n"
                 part << "#{value}\r\n"
@@ -76,9 +76,9 @@ module Datadog
               # @param io [IO]
               # @param headers [Hash]
               def initialize(boundary, name, io, headers = {})
-                file_length = io.respond_to?(:length) ?  io.length : File.size(io.local_path)
+                file_length = io.respond_to?(:length) ? io.length : File.size(io.local_path)
                 @head = build_head(boundary, name, io.original_filename, io.content_type, file_length,
-                                  io.respond_to?(:opts) ? io.opts.merge(headers) : headers)
+                  io.respond_to?(:opts) ? io.opts.merge(headers) : headers)
                 @foot = "\r\n"
                 @length = @head.bytesize + file_length + @foot.length
                 @io = CompositeReadIO.new(StringIO.new(@head), io, StringIO.new(@foot))
@@ -98,16 +98,16 @@ module Datadog
 
                 part = ''
                 part << "--#{boundary}\r\n"
-                part << "Content-Disposition: #{content_disposition}; name=\"#{name.to_s}\"; filename=\"#{filename}\"\r\n"
+                part << "Content-Disposition: #{content_disposition}; name=\"#{name}\"; filename=\"#{filename}\"\r\n"
                 part << "Content-Length: #{content_len}\r\n"
                 if content_id = opts.delete("Content-ID")
                   part << "Content-ID: #{content_id}\r\n"
                 end
 
-                if opts["Content-Type"] != nil
-                  part <<  "Content-Type: " + opts["Content-Type"] + "\r\n"
+                part << if !opts["Content-Type"].nil?
+                  "Content-Type: " + opts["Content-Type"] + "\r\n"
                 else
-                  part << "Content-Type: #{type}\r\n"
+                  "Content-Type: #{type}\r\n"
                 end
 
                 part << "Content-Transfer-Encoding: #{trans_encoding}\r\n"

data/lib/datadog/core/worker.rb

@@ -12,7 +12,7 @@ module Datadog
       end
 
       def perform(*args)
-        task.call(*args) unless task.nil?
+        task&.call(*args)
       end
 
       protected

data/lib/datadog/core/workers/async.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative '../logger'
+require_relative "../logger"
 
 module Datadog
   module Core
@@ -8,6 +8,11 @@ module Datadog
       module Async
         # Adds threading behavior to workers
         # to run tasks asynchronously.
+        #
+        # This module is included in Polling module, and has no other
+        # direct users.
+        #
+        # @api private
         module Thread
           FORK_POLICY_STOP = :stop
           FORK_POLICY_RESTART = :restart
@@ -24,7 +29,11 @@ module Datadog
           # Methods that must be prepended
           module PrependedMethods
             def perform(*args)
-              start_async { self.result = super(*args) } unless started?
+              unless started?
+                start_async do
+                  self.result = super(*args)
+                end
+              end
             end
           end
 
@@ -47,6 +56,14 @@ module Datadog
             @run_async = false
             Datadog.logger.debug { "Forcibly terminating worker thread for: #{self}" }
             worker.terminate
+            # Wait for the worker thread to end
+            begin
+              Timeout.timeout(SHUTDOWN_TIMEOUT) do
+                worker.join
+              end
+            rescue Timeout::Error
+              Datadog.logger.debug { "Worker thread did not end after #{SHUTDOWN_TIMEOUT} seconds: #{self}" }
+            end
             true
           end
 
@@ -112,23 +129,33 @@ module Datadog
             @worker ||= nil
           end
 
+          # Returns true if worker thread is successfully started,
+          # false if it is not started. Reasons for not starting the worker
+          # thread: it is already running, or the process forked and fork
+          # policy is to stop the workers on fork (which means they are
+          # not started in children, really).
           def start_async(&block)
             mutex.synchronize do
-              return if running?
+              return false if running?
 
               if forked?
                 case fork_policy
                 when FORK_POLICY_STOP
                   stop_fork
+                  false
                 when FORK_POLICY_RESTART
+                  # restart_after_fork should return true
                   restart_after_fork(&block)
                 end
               elsif !run_async?
+                # start_worker should return true
                 start_worker(&block)
               end
             end
           end
 
+          # Returns true if worker thread is successfully started,
+          # which should generally be always.
           def start_worker
             @run_async = true
             @pid = Process.pid
@@ -136,22 +163,21 @@ module Datadog
             Datadog.logger.debug { "Starting thread for: #{self}" }
 
             @worker = ::Thread.new do
-              begin
-                yield
-              # rubocop:disable Lint/RescueException
-              rescue Exception => e
-                @error = e
-                Datadog.logger.debug(
-                  "Worker thread error. Cause: #{e.class.name} #{e.message} Location: #{Array(e.backtrace).first}"
-                )
-                raise
-              end
+              yield
+            # rubocop:disable Lint/RescueException
+            rescue Exception => e
+              @error = e
+              Datadog.logger.debug(
+                "Worker thread error. Cause: #{e.class}: #{e.message} Location: #{Array(e.backtrace).first}"
+              )
+              raise
+
               # rubocop:enable Lint/RescueException
             end
             @worker.name = self.class.name
             @worker.thread_variable_set(:fork_safe, true)
 
-            nil
+            true
           end
 
           def stop_fork

data/lib/datadog/core/workers/interval_loop.rb

@@ -5,6 +5,11 @@ module Datadog
     module Workers
       # Adds looping behavior to workers, with a sleep
       # interval between each loop.
+      #
+      # This module is included in Polling module, and has no other
+      # direct users.
+      #
+      # @api private
       module IntervalLoop
         BACK_OFF_RATIO = 1.2
         BACK_OFF_MAX = 5
@@ -21,25 +26,67 @@ module Datadog
         # Methods that must be prepended
         module PrependedMethods
           def perform(*args)
-            perform_loop { super(*args) }
+            perform_loop do
+              @in_iteration = true
+              begin
+                super(*args)
+              ensure
+                @in_iteration = false
+              end
+            end
+          end
+
+          def in_iteration?
+            defined?(@in_iteration) && @in_iteration
           end
         end
 
         def stop_loop
           mutex.synchronize do
-            return false unless run_loop?
-
+            # Do not call run_loop? from this method to see if the loop
+            # is running, because @run_loop is normally initialized by
+            # the background thread and if the stop is requested right
+            # after the worker starts, the background thread may be created
+            # (and scheduled) but hasn't run yet, thus skipping the
+            # write to @run_loop here would leave the thread running forever.
             @run_loop = false
+
+            # It is possible that we don't need to signal shutdown if
+            # @run_loop was not initialized (i.e. we changed it from not
+            # defined to false above). But let's be safe and signal the
+            # shutdown anyway, I don't see what harm it can cause.
             shutdown.signal
           end
 
+          # Previously, this method would return false (and do nothing)
+          # if the worker was not running the loop. However, this was racy -
+          # see https://github.com/DataDog/ruby-guild/issues/279.
+          # stop_loop now always sets the state to "stop requested" and,
+          # correspondingly, always returns true.
+          #
+          # There is some test code that returns false when mocking this
+          # method - most likely this method should be treated as a void one
+          # and the caller should assume that the stop was always requested.
           true
         end
 
+        # TODO: This method carries two semantics today:
+        # 1. Is there work available to process?
+        # 2. Should the main loop keep running?
+        # Things get messy when `Queue` is mixed in,
+        # wanting to override semantic 1 but not 2.
+        # This should probably be split into two methods:
+        # `work_pending?` (semantic 1) and `continue_loop?` (semantic 2).
+        # Today, `run_loop?` performs semantic 2, but should probably be
+        # renamed to `continue_loop?`, since that would make it clear that
+        # we're just checking if we can "keep going inside the loop",
+        # and not checking if we should run the loop right now.
+        # Clean up {Workers::Queue} after this.
         def work_pending?
           run_loop?
         end
 
+        # TODO: Probably should be renamed to `continue_loop?`, see `work_pending?` TODO.
         def run_loop?
           return false unless instance_variable_defined?(:@run_loop)
 
@@ -93,7 +140,19 @@ module Datadog
 
         def perform_loop
           mutex.synchronize do
-            @run_loop = true
+            unless defined?(@run_loop)
+              # This write must only happen if @run_loop is not defined
+              # (i.e., not initialized). In the case when the worker is
+              # asked to stop right after it is created, the thread may not
+              # have run yet by the time +stop_loop+ is invoked and
+              # we need to preserve the stop-requested state from
+              # +stop_loop+ to +perform_loop+.
+              #
+              # If the workers are refactored to use classes and inheritance
+              # and their state, such as @run_loop, is initialized in
+              # constructors, the write can be made unconditional.
+              @run_loop = true
+            end
 
             shutdown.wait(mutex, loop_wait_time) if loop_wait_before_first_iteration?
           end

data/lib/datadog/core/workers/polling.rb

@@ -1,12 +1,14 @@
 # frozen_string_literal: true
 
-require_relative 'async'
-require_relative 'interval_loop'
+require_relative "async"
+require_relative "interval_loop"
 
 module Datadog
   module Core
     module Workers
       # Adds polling (async looping) behavior to workers
+      #
+      # @api private
       module Polling
         DEFAULT_SHUTDOWN_TIMEOUT = 1
 

data/lib/datadog/core/workers/queue.rb

@@ -5,6 +5,17 @@ module Datadog
     module Workers
       # Adds queue behavior to workers, with a buffer
       # to which items can be queued then dequeued.
+      #
+      # This module is included in some but not all workers.
+      # Notably, Data Streams Processor uses a queue but implements it
+      # inline rather than using this module.
+      #
+      # The workers that do include Queue also include Polling, which
+      # in turn includes Async::Thread and IntervalLoop. This means
+      # we have e.g. +in_iteration?+ always available in any worker
+      # that includes Queue.
+      #
+      # @api private
       module Queue
         def self.included(base)
           base.prepend(PrependedMethods)
@@ -13,11 +24,16 @@ module Datadog
         # Methods that must be prepended
         module PrependedMethods
           def perform(*args)
-            super(*dequeue) if work_pending?
+            if work_pending?
+              work = dequeue
+              super(*work)
+            end
           end
         end
 
         def buffer
+          # Why is this an unsynchronized Array and not a Core::Buffer
+          # instance?
           @buffer ||= []
         end
 
@@ -34,10 +50,89 @@ module Datadog
           !buffer.empty?
         end
 
+        # Wait for the worker to finish handling all work that has already
+        # been submitted to it.
+        #
+        # If the worker is not enabled, returns nil.
+        # If the worker is enabled, returns whether, at the point of return,
+        # there was no pending or in progress work.
+        #
+        # Flushing can time out because there is a constant stream of work
+        # submitted at the same or higher rate than it is processed.
+        # Flushing can also fail if the worker thread is not running -
+        # this method will not flush from the calling thread.
+        def flush(timeout: nil)
+          # Default timeout is 5 seconds.
+          # Specific workers can override it to be more or less
+          timeout ||= 5
+
+          # Nothing needs to be done if the worker is not enabled.
+          return nil unless enabled?
+
+          unless running?
+            unless buffer.empty?
+              # If we are asked to flush but the worker is not running,
+              # do not flush from the caller thread. If the buffer is not
+              # empty, it will not be flushed. Log a warning to this effect.
+              #
+              # We are not guaranteed to have a logger as an instance method,
+              # reference the global for now - all other worker methods
+              # also reference the logger globally.
+              # TODO inject it into worker instances.
+              Datadog.logger.debug { "Asked to flush #{self} when the worker is not running" }
+              return false
+            end
+          end
+
+          started = Utils::Time.get_time
+          loop do
+            # The AppStarted event is triggered by the worker itself,
+            # from the worker thread. As such the main thread has no way
+            # to delay itself until that event is queued and we need some
+            # way to wait until that event is sent out to assert on it in
+            # the test suite. Check the run once flag which *should*
+            # indicate the event has been queued (at which point our queue
+            # depth check should wait until it's sent).
+            # This is still a hack because the flag can be overridden
+            # either way with or without the event being sent out.
+            # Note that if the AppStarted sending fails, this check
+            # will return false and flushing will be blocked until the
+            # 15 second timeout.
+            # Note that the first wait interval between telemetry event
+            # sending is 10 seconds, the timeout needs to be strictly
+            # greater than that.
+            return true if idle?
+
+            return false if Utils::Time.get_time - started > timeout
+
+            sleep 0.5
+          end
+        end
+
         protected
 
         attr_writer \
           :buffer
+
+        # Returns whether this worker has no pending work and is not actively
+        # working.
+        #
+        # The reason why "actively working" is considered is that we use
+        # flushing to ensure all work is completed before asserting on the
+        # outcome in the tests - if work is happening in a background thread,
+        # it's too early to assert on its results.
+        def idle?
+          # We have a +work_pending?+ method in this class that semantically
+          # would be appropriate here instead of calling +buffer.empty?+.
+          # Unfortunately IntervalLoop replaces our implementation of
+          # +work_pending?+ with one that doesn't make sense at least for the
+          # Queue. And we can't change the order of module includes because
+          # they all override +perform+ and the correct behavior depends on
+          # placing IntervalLoop after Queue.
+          #
+          # Until this mess is untangled, call +buffer.empty?+ here.
+          buffer.empty? && !in_iteration?
+        end
       end
     end
   end

data/lib/datadog/core/workers/runtime_metrics.rb

@@ -1,10 +1,10 @@
 # frozen_string_literal: true
 
-require_relative '../runtime/metrics'
+require_relative "../runtime/metrics"
 
-require_relative '../worker'
-require_relative 'async'
-require_relative 'polling'
+require_relative "../worker"
+require_relative "async"
+require_relative "polling"
 
 module Datadog
   module Core
@@ -20,8 +20,16 @@ module Datadog
         attr_reader \
           :metrics
 
-        def initialize(options = {})
-          @metrics = options.fetch(:metrics) { Core::Runtime::Metrics.new }
+        def initialize(telemetry:, **options)
+          @metrics = options.fetch(:metrics) do
+            Core::Runtime::Metrics.new(
+              logger: options[:logger],
+              telemetry: telemetry,
+              experimental_propagate_process_tags_enabled: options.fetch(:propagate_process_tags_enabled) do
+                options.fetch(:experimental_propagate_process_tags_enabled)
+              end
+            )
+          end
 
           # Workers::Async::Thread settings
           self.fork_policy = options.fetch(:fork_policy, Workers::Async::Thread::FORK_POLICY_STOP)

data/lib/datadog/core.rb

@@ -1,7 +1,11 @@
 # frozen_string_literal: true
 
-require_relative 'core/deprecations'
-require_relative 'core/extensions'
+# Keep this at the top, this is needed at require-time by some files
+require_relative "ruby_version"
+
+require_relative "core/deprecations"
+require_relative "core/configuration/config_helper"
+require_relative "core/extensions"
 
 # We must load core extensions to make certain global APIs
 # accessible: both for Datadog features and the core itself.
@@ -11,6 +15,14 @@ module Datadog
   # for higher-level features.
   module Core
     extend Core::Deprecations
+
+    LIBDATADOG_API_FAILURE =
+      begin
+        require "libdatadog_api.#{RUBY_VERSION[/\d+.\d+/]}_#{RUBY_PLATFORM}"
+        nil
+      rescue LoadError => e
+        "#{e.class}: #{e.message}"
+      end
   end
 
   extend Core::Extensions
@@ -19,9 +31,14 @@ module Datadog
   # Ensures the Datadog components have a chance to gracefully
   # shut down and cleanup before terminating the process.
   at_exit do
-    if Interrupt === $! # rubocop:disable Style/SpecialGlobalVars is process terminating due to a ctrl+c or similar?
+    exception = $! # rubocop:disable Style/SpecialGlobalVars
+
+    if Interrupt === exception # is process terminating due to a ctrl+c or similar?
       Datadog.send(:handle_interrupt_shutdown!)
     else
+      # Report unhandled exception to crash tracker before shutdown
+      Datadog::Core::Crashtracking::Component.report_unhandled_exception(exception)
+
       Datadog.shutdown!
     end
   end

data/lib/datadog/data_streams/configuration.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require_relative "../core/environment/variable_helpers"
+require_relative "ext"
+
+module Datadog
+  module DataStreams
+    # Configuration for Data Streams Monitoring
+    module Configuration
+      # Configuration settings for Data Streams Monitoring.
+      module Settings
+        def self.extended(base)
+          base = base.singleton_class unless base.is_a?(Class)
+          add_settings!(base)
+        end
+
+        def self.add_settings!(base)
+          base.class_eval do
+            # Data Streams Monitoring configuration
+            # @public_api
+            settings :data_streams do
+              # Whether Data Streams Monitoring is enabled. When enabled, the library will
+              # collect and report data lineage information for messaging systems.
+              #
+              # @default `DD_DATA_STREAMS_ENABLED` environment variable, otherwise `false`.
+              # @return [Boolean]
+              option :enabled do |o|
+                o.type :bool
+                o.env Ext::ENV_ENABLED
+                o.default false
+              end
+
+              # The interval (in seconds) at which Data Streams Monitoring stats are flushed.
+              #
+              # @default 10.0
+              # @env '_DD_TRACE_STATS_WRITER_INTERVAL'
+              # @return [Float]
+              # @!visibility private
+              option :interval do |o|
+                o.type :float
+                o.env "_DD_TRACE_STATS_WRITER_INTERVAL"
+                o.default 10.0
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/data_streams/ext.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DataStreams
+    # Constants for Data Streams Monitoring configuration
+    # @public_api
+    module Ext
+      ENV_ENABLED = "DD_DATA_STREAMS_ENABLED"
+    end
+  end
+end