datadog 2.23.0 → 2.24.0

data/lib/datadog/core/utils/time.rb

@@ -10,7 +10,7 @@ module Datadog
         # Current monotonic time
         #
         # @param unit [Symbol] unit for the resulting value, same as ::Process#clock_gettime, defaults to :float_second
-        # @return [Numeric] timestamp in the requested unit, since some unspecified starting point
+        # @return [Float|Integer] timestamp in the requested unit, since some unspecified starting point
         def get_time(unit = :float_second)
           Process.clock_gettime(Process::CLOCK_MONOTONIC, unit)
         end

data/lib/datadog/core/workers/async.rb

@@ -8,6 +8,11 @@ module Datadog
       module Async
         # Adds threading behavior to workers
         # to run tasks asynchronously.
+        #
+        # This module is included in Polling module, and has no other
+        # direct users.
+        #
+        # @api private
         module Thread
           FORK_POLICY_STOP = :stop
           FORK_POLICY_RESTART = :restart
@@ -24,7 +29,11 @@ module Datadog
           # Methods that must be prepended
           module PrependedMethods
             def perform(*args)
-              start_async { self.result = super(*args) } unless started?
+              unless started?
+                start_async do
+                  self.result = super(*args)
+                end
+              end
             end
           end
 

data/lib/datadog/core/workers/interval_loop.rb

@@ -5,6 +5,11 @@ module Datadog
     module Workers
       # Adds looping behavior to workers, with a sleep
       # interval between each loop.
+      #
+      # This module is included in Polling module, and has no other
+      # direct users.
+      #
+      # @api private
       module IntervalLoop
         BACK_OFF_RATIO = 1.2
         BACK_OFF_MAX = 5
@@ -38,15 +43,39 @@ module Datadog
 
         def stop_loop
           mutex.synchronize do
-            return false unless run_loop?
-
+            # Do not call run_loop? from this method to see if the loop
+            # is running, because @run_loop is normally initialized by
+            # the background thread and if the stop is requested right
+            # after the worker starts, the background thread may be created
+            # (and scheduled) but hasn't run yet, thus skipping the
+            # write to @run_loop here would leave the thread running forever.
             @run_loop = false
+
+            # It is possible that we don't need to signal shutdown if
+            # @run_loop was not initialized (i.e. we changed it from not
+            # defined to false above). But let's be safe and signal the
+            # shutdown anyway, I don't see what harm it can cause.
             shutdown.signal
           end
 
+          # Previously, this method would return false (and do nothing)
+          # if the worker was not running the loop. However, this was racy -
+          # see https://github.com/DataDog/ruby-guild/issues/279.
+          # stop_loop now always sets the state to "stop requested" and,
+          # correspondingly, always returns true.
+          #
+          # There is some test code that returns false when mocking this
+          # method - most likely this method should be treated as a void one
+          # and the caller should assume that the stop was always requested.
           true
         end
 
+        # TODO This overwrites Queue's +work_pending?+ method with an
+        # implementation that, to me, is at leat questionable semantically:
+        # the Queue's idea of pending work is if the buffer is not empty,
+        # but this module says that work is pending if the work processing
+        # loop is scheduled to run (in other words, as long as the background
+        # thread is running, there is always pending work).
         def work_pending?
           run_loop?
         end
@@ -104,7 +133,19 @@ module Datadog
 
         def perform_loop
           mutex.synchronize do
-            @run_loop = true
+            unless defined?(@run_loop)
+              # This write must only happen if @run_loop is not defined
+              # (i.e., not initialized). In the case when the worker is
+              # asked to stop right after it is created, the thread may not
+              # have run yet by the time +stop_loop+ is invoked and
+              # we need to preserve the stop-requested state from
+              # +stop_loop+ to +perform_loop+.
+              #
+              # If the workers are refactored to use classes and inheritance
+              # and their state, such as @run_loop, is initialized in
+              # constructors, the write can be made unconditional.
+              @run_loop = true
+            end
 
             shutdown.wait(mutex, loop_wait_time) if loop_wait_before_first_iteration?
           end

data/lib/datadog/core/workers/polling.rb

@@ -7,6 +7,8 @@ module Datadog
   module Core
     module Workers
       # Adds polling (async looping) behavior to workers
+      #
+      # @api private
       module Polling
         DEFAULT_SHUTDOWN_TIMEOUT = 1
 

data/lib/datadog/core/workers/queue.rb

@@ -5,6 +5,17 @@ module Datadog
     module Workers
       # Adds queue behavior to workers, with a buffer
       # to which items can be queued then dequeued.
+      #
+      # This module is included in some but not all workers.
+      # Notably, Data Streams Processor uses a queue but implements it
+      # inline rather than using this module.
+      #
+      # The workers that do include Queue also include Polling, which
+      # in turn includes Async::Thread and IntervalLoop. This means
+      # we have e.g. +in_iteration?+ always available in any worker
+      # that includes Queue.
+      #
+      # @api private
       module Queue
         def self.included(base)
           base.prepend(PrependedMethods)
@@ -13,11 +24,16 @@ module Datadog
         # Methods that must be prepended
         module PrependedMethods
           def perform(*args)
-            super(*dequeue) if work_pending?
+            if work_pending?
+              work = dequeue
+              super(*work)
+            end
           end
         end
 
         def buffer
+          # Why is this an unsynchronized Array and not a Core::Buffer
+          # instance?
           @buffer ||= []
         end
 
@@ -34,10 +50,93 @@ module Datadog
           !buffer.empty?
         end
 
+        # Wait for the worker to finish handling all work that has already
+        # been submitted to it.
+        #
+        # If the worker is not enabled, returns nil.
+        # If the worker is enabled, returns whether, at the point of return,
+        # there was no pending or in progress work.
+        #
+        # Flushing can time out because there is a constant stream of work
+        # submitted at the same or higher rate than it is processed.
+        # Flushing can also fail if the worker thread is not running -
+        # this method will not flush from the calling thread.
+        def flush(timeout: nil)
+          # Default timeout is 5 seconds.
+          # Specific workers can override it to be more or less
+          timeout ||= 5
+
+          # Nothing needs to be done if the worker is not enabled.
+          return nil unless enabled?
+
+          unless running?
+            unless buffer.empty?
+              # If we are asked to flush but the worker is not running,
+              # do not flush from the caller thread. If the buffer is not
+              # empty, it will not be flushed. Log a warning to this effect.
+              #
+              # We are not guaranteed to have a logger as an instance method,
+              # reference the global for now - all other worker methods
+              # also reference the logger globally.
+              # TODO inject it into worker instances.
+              Datadog.logger.debug { "Asked to flush #{self} when the worker is not running" }
+              return false
+            end
+          end
+
+          started = Utils::Time.get_time
+          loop do
+            # The AppStarted event is triggered by the worker itself,
+            # from the worker thread. As such the main thread has no way
+            # to delay itself until that event is queued and we need some
+            # way to wait until that event is sent out to assert on it in
+            # the test suite. Check the run once flag which *should*
+            # indicate the event has been queued (at which point our queue
+            # depth check should wait until it's sent).
+            # This is still a hack because the flag can be overridden
+            # either way with or without the event being sent out.
+            # Note that if the AppStarted sending fails, this check
+            # will return false and flushing will be blocked until the
+            # 15 second timeout.
+            # Note that the first wait interval between telemetry event
+            # sending is 10 seconds, the timeout needs to be strictly
+            # greater than that.
+            return true if idle?
+
+            return false if Utils::Time.get_time - started > timeout
+
+            sleep 0.5
+          end
+        end
+
         protected
 
         attr_writer \
           :buffer
+
+        # Returns whether this worker has no pending work and is not actively
+        # working.
+        #
+        # The reason why "actively working" is considered is that we use
+        # flushing to ensure all work is completed before asserting on the
+        # outcome in the tests - if work is happening in a background thread,
+        # it's too early to assert on its results.
+        def idle?
+          # We have a +work_pending?+ method in this class that semantically
+          # would be appropriate here instead of calling +buffer.empty?+.
+          # Unfortunately IntervalLoop replaces our implementation of
+          # +work_pending?+ with one that doesn't make sense at least for the
+          # Queue. And we can't change the order of module includes because
+          # they all override +perform+ and the correct behavior depends on
+          # placing IntervalLoop after Queue.
+          #
+          # The TraceWriter worker then defines +work_pending?+ to be the
+          # same as Queue implementation here... Essentially, it demands
+          # the behavior that perhaps should be applied to all workers.
+          #
+          # Until this mess is untangled, call +buffer.empty?+ here.
+          buffer.empty? && !in_iteration?
+        end
       end
     end
   end

data/lib/datadog/data_streams/processor.rb

@@ -242,7 +242,7 @@ module Datadog
         current_context = get_current_context
         tags = tags.sort
 
-        direction = nil
+        direction = nil #: ::String?
         tags.each do |tag|
           if tag.start_with?('direction:')
             direction = tag

data/lib/datadog/data_streams/transport/http/stats.rb

@@ -1,11 +1,8 @@
 # frozen_string_literal: true
 
 require_relative '../stats'
-require_relative 'client'
-require_relative '../../../core/transport/http/response'
 require_relative '../../../core/transport/http/api/endpoint'
-require_relative '../../../core/transport/http/api/spec'
-require_relative '../../../core/transport/http/api/instance'
+require_relative '../../../core/transport/http/response'
 
 module Datadog
   module DataStreams
@@ -23,38 +20,6 @@ module Datadog
           end
 
           module API
-            # HTTP API Spec for DSM
-            class Spec < Core::Transport::HTTP::API::Spec
-              attr_accessor :stats
-
-              def send_stats(env, &block)
-                raise Core::Transport::HTTP::API::Spec::EndpointNotDefinedError.new('stats', self) if stats.nil?
-
-                stats.call(env, &block)
-              end
-
-              def encoder
-                # DSM handles encoding in the transport layer (MessagePack + gzip)
-                # so we don't need an encoder at the API level
-                nil
-              end
-            end
-
-            # HTTP API Instance for DSM
-            class Instance < Core::Transport::HTTP::API::Instance
-              def send_stats(env)
-                unless spec.is_a?(Stats::API::Spec)
-                  raise Core::Transport::HTTP::API::Instance::EndpointNotSupportedError.new(
-                    'stats', self
-                  )
-                end
-
-                spec.send_stats(env) do |request_env|
-                  call(request_env)
-                end
-              end
-            end
-
             # Endpoint for submitting DSM stats data
             class Endpoint < Core::Transport::HTTP::API::Endpoint
               def initialize(path)

data/lib/datadog/data_streams/transport/http.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
 require_relative '../../core/transport/http'
-require_relative 'http/api'
-require_relative 'http/client'
 require_relative 'http/stats'
 require_relative 'stats'
 
@@ -11,6 +9,10 @@ module Datadog
     module Transport
       # HTTP transport for Data Streams Monitoring
       module HTTP
+        V01 = Stats::API::Endpoint.new(
+          '/v0.1/pipeline_stats'
+        )
+
         module_function
 
         # Builds a new Transport::HTTP::Client with default settings
@@ -19,7 +21,6 @@ module Datadog
           logger:
         )
           Core::Transport::HTTP.build(
-            api_instance_class: Stats::API::Instance,
             agent_settings: agent_settings,
             logger: logger,
             headers: {
@@ -27,9 +28,7 @@ module Datadog
               'Content-Encoding' => 'gzip'
             }
           ) do |transport|
-            apis = API.defaults
-
-            transport.api API::V01, apis[API::V01], default: true
+            transport.api 'v0.1', V01, default: true
 
             # Call block to apply any customization, if provided
             yield(transport) if block_given?

data/lib/datadog/data_streams/transport/stats.rb

@@ -4,6 +4,7 @@ require 'msgpack'
 require 'zlib'
 require_relative '../../core/transport/parcel'
 require_relative '../../core/transport/request'
+require_relative '../../core/transport/transport'
 
 module Datadog
   module DataStreams
@@ -25,18 +26,7 @@ module Datadog
         end
 
         # Transport for Data Streams Monitoring stats
-        class Transport
-          attr_reader :client, :apis, :current_api_id, :logger
-
-          def initialize(apis, default_api, logger:)
-            @apis = apis
-            @logger = logger
-            @default_api = default_api
-            @current_api_id = default_api
-
-            @client = DataStreams::Transport::HTTP::Client.new(current_api, logger: @logger)
-          end
-
+        class Transport < Core::Transport::Transport
           def send_stats(payload)
             # MessagePack encode and gzip compress the payload
             msgpack_data = MessagePack.pack(payload)
@@ -47,11 +37,7 @@ module Datadog
             request = Request.new(parcel)
 
             # Send to agent
-            client.send_stats_payload(request)
-          end
-
-          def current_api
-            apis[@current_api_id]
+            client.send_request(:stats, request)
           end
         end
       end

data/lib/datadog/di/contrib/active_record.rb

@@ -1,12 +1,38 @@
 # frozen_string_literal: true
 
-Datadog::DI::Serializer.register(condition: lambda { |value| ActiveRecord::Base === value }) do |serializer, value, name:, depth:| # steep:ignore
-  # steep thinks all of the arguments are nil here
-  # steep:ignore:start
+# steep thinks all of the arguments are nil here and does not know what ActiveRecord is.
+# steep:ignore:start
+
+Datadog::DI::Serializer.register(
+  # This serializer uses a dynamic condition to determine its applicability
+  # to a particular value. A simpler case could have been a serializer for
+  # a particular class, but in this case any ActiveRecord model is covered
+  # and they all have different classes.
+  #
+  # An alternative could have been to make DI specifically provide lookup
+  # logic for "instances of classes derived from X", but a condition Proc
+  # is more universal.
+  condition: lambda { |value| ActiveRecord::Base === value }
+) do |serializer, value, name:, depth:|
+  # +serializer+ is an instance of DI::Serializer.
+  # Use it to perform the serialization to primitive values.
+  #
+  # +value+ is the value to serialize. It should match the condition
+  # provided above, meaning it would be an ActiveRecord::Base instance.
+  #
+  # +name+ is the name of the (local/instance) variable being serialized.
+  # The name is used by DI for redaction (upstream of serialization logic),
+  # and could potentially be used for redaction here also.
+  #
+  # +depth+ is the remaining depth for serializing collections and objects.
+  # It should always be an integer.
+  # Reduce it by 1 when invoking +serialize_value+ on the contents of +value+.
+  # This serializer could also potentially do its own depth limiting.
   value_to_serialize = {
     attributes: value.attributes,
     new_record: value.new_record?,
   }
-  serializer.serialize_value(value_to_serialize, depth: depth ? depth - 1 : nil, type: value.class)
-  # steep:ignore:end
+  serializer.serialize_value(value_to_serialize, depth: depth - 1, type: value.class)
 end
+
+# steep:ignore:end

data/lib/datadog/di/el/compiler.rb

@@ -22,7 +22,8 @@ module Datadog
 
         private
 
-        OPERATORS = {
+        # Steep: https://github.com/soutaro/steep/issues/363
+        OPERATORS = { # steep:ignore IncompatibleAssignment
           'eq' => '==',
           'ne' => '!=',
           'ge' => '>=',
@@ -31,16 +32,19 @@ module Datadog
           'lt' => '<',
         }.freeze
 
+        # Steep: https://github.com/soutaro/steep/issues/363
         SINGLE_ARG_METHODS = %w[
           len isEmpty isUndefined
-        ].freeze
+        ].freeze # steep:ignore IncompatibleAssignment
 
+        # Steep: https://github.com/soutaro/steep/issues/363
         TWO_ARG_METHODS = %w[
           startsWith endsWith contains matches
           getmember index instanceof
-        ].freeze
+        ].freeze # steep:ignore IncompatibleAssignment
 
-        MULTI_ARG_METHODS = {
+        # Steep: https://github.com/soutaro/steep/issues/363
+        MULTI_ARG_METHODS = { # steep:ignore IncompatibleAssignment
           'and' => '&&',
           'or' => '||',
         }.freeze

data/lib/datadog/di/error.rb

@@ -42,6 +42,11 @@ module Datadog
       class ProbePreviouslyFailed < Error
       end
 
+      # Raised when trying to instrument a probe when there is existing
+      # instrumentation for the same probe id.
+      class AlreadyInstrumented < Error
+      end
+
       # Raised when installing a line probe and multiple files match the
       # specified path suffix.
       # A probe must be installed into one file only, since UI only

data/lib/datadog/di/instrumenter.rb

@@ -166,7 +166,10 @@ module Datadog
                   depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
                   attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count)
               end
-              start_time = Core::Utils::Time.get_time
+              # We intentionally do not use Core::Utils::Time.get_time
+              # here because the time provider may be overridden by the
+              # customer, and DI is not allowed to invoke customer code.
+              start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
 
               rv = nil
               begin
@@ -190,7 +193,7 @@ module Datadog
                 # the instrumentation callback runs.
               end
 
-              duration = Core::Utils::Time.get_time - start_time
+              duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
               # The method itself is not part of the stack trace because
               # we are getting the stack trace from outside of the method.
               # Add the method in manually as the top frame.
@@ -256,6 +259,8 @@ module Datadog
 
           probe.instrumentation_module = mod
           cls.send(:prepend, mod)
+
+          DI.instrumented_count_inc(:method)
         end
       end
 
@@ -268,6 +273,8 @@ module Datadog
           if mod = probe.instrumentation_module
             mod.send(:remove_method, probe.method_name)
             probe.instrumentation_module = nil
+
+            DI.instrumented_count_dec(:method)
           end
         end
       end
@@ -470,12 +477,16 @@ module Datadog
           # actual_path could be nil if we don't use targeted trace points.
           probe.instrumented_path = actual_path
 
-          if iseq
+          # TracePoint#enable returns false when it succeeds.
+          rv = if iseq
             tp.enable(target: iseq, target_line: line_no)
           else
             tp.enable
           end
-          # TracePoint#enable returns false when it succeeds.
+
+          DI.instrumented_count_inc(:line)
+
+          rv
         end
         true
       end
@@ -485,6 +496,8 @@ module Datadog
           if tp = probe.instrumentation_trace_point
             tp.disable
             probe.instrumentation_trace_point = nil
+
+            DI.instrumented_count_dec(:line)
           end
         end
       end

data/lib/datadog/di/probe_builder.rb

@@ -20,7 +20,8 @@ module Datadog
     #
     # @api private
     module ProbeBuilder
-      PROBE_TYPES = {
+      # Steep: https://github.com/soutaro/steep/issues/363
+      PROBE_TYPES = { # steep:ignore IncompatibleAssignment
         'LOG_PROBE' => :log,
       }.freeze
 

data/lib/datadog/di/probe_manager.rb

@@ -94,6 +94,19 @@ module Datadog
       # matches.
       def add_probe(probe)
         @lock.synchronize do
+          if @installed_probes[probe.id]
+            # Either this probe was already installed, or another probe was
+            # installed with the same id (previous version perhaps?).
+            # Since our state tracking is keyed by probe id, we cannot
+            # install this probe since we won't have a way of removing the
+            # instrumentation for the probe with the same id which is already
+            # installed.
+            #
+            # The exception raised here will be caught below and logged and
+            # reported to telemetry.
+            raise Error::AlreadyInstrumented, "Probe with id #{probe.id} is already in installed probes"
+          end
+
           # Probe failed to install previously, do not try to install it again.
           if msg = @failed_probes[probe.id]
             # TODO test this path
@@ -134,38 +147,30 @@ module Datadog
         end
       end
 
-      # Removes probes with ids other than in the specified list.
-      #
-      # This method is meant to be invoked from remote config processor.
-      # Remote config contains the list of currently defined probes; any
-      # probes not in that list have been removed by user and should be
-      # de-instrumented from the application.
-      def remove_other_probes(probe_ids)
+      # Removes probe with specified id. The probe could be pending or
+      # installed. Does nothing if there is no probe with the specified id.
+      def remove_probe(probe_id)
         @lock.synchronize do
-          @pending_probes.values.each do |probe|
-            unless probe_ids.include?(probe.id)
-              @pending_probes.delete(probe.id)
-            end
-          end
-          @installed_probes.values.each do |probe|
-            unless probe_ids.include?(probe.id)
-              begin
-                instrumenter.unhook(probe)
-                # Only remove the probe from installed list if it was
-                # successfully de-instrumented. Active probes do incur overhead
-                # for the running application, and if the error is ephemeral
-                # we want to try removing the probe again at the next opportunity.
-                #
-                # TODO give up after some time?
-                @installed_probes.delete(probe.id)
-              rescue => exc
-                raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
-                # Silence all exceptions?
-                # TODO should we propagate here and rescue upstream?
-                logger.debug { "di: error removing #{probe.type} probe at #{probe.location} (#{probe.id}): #{exc.class}: #{exc}" }
-                telemetry&.report(exc, description: "Error removing probe")
-              end
-            end
+          @pending_probes.delete(probe_id)
+        end
+
+        # Do not delete the probe from the registry here in case
+        # deinstrumentation fails - though I don't know why deinstrumentation
+        # would fail and how we could recover if it does.
+        # I plan on tracking the number of outstanding (instrumented) probes
+        # in the future, and if deinstrumentation fails I would want to
+        # keep that probe as "installed" for the count, so that we can
+        # investigate the situation.
+        if probe = @installed_probes[probe_id]
+          begin
+            instrumenter.unhook(probe)
+            @installed_probes.delete(probe_id)
+          rescue => exc
+            raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
+            # Silence all exceptions?
+            # TODO should we propagate here and rescue upstream?
+            logger.debug { "di: error removing #{probe.type} probe at #{probe.location} (#{probe.id}): #{exc.class}: #{exc}" }
+            telemetry&.report(exc, description: "Error removing probe")
           end
         end
       end
@@ -185,6 +190,7 @@ module Datadog
                   # TODO is it OK to hook from trace point handler?
                   # TODO the class is now defined, but can hooking still fail?
                   instrumenter.hook(probe, self)
+                  @installed_probes[probe.id] = probe
                   @pending_probes.delete(probe.id)
                   break
                 rescue Error::DITargetNotDefined