datadog 2.22.0 → 2.26.0

data/lib/datadog/data_streams/transport/http/stats.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require_relative '../stats'
+require_relative '../../../core/transport/http/api/endpoint'
+require_relative '../../../core/transport/http/response'
+
+module Datadog
+  module DataStreams
+    module Transport
+      module HTTP
+        # HTTP transport behavior for Data Streams stats
+        module Stats
+          # Response from HTTP transport for DSM stats
+          class Response
+            include Datadog::Core::Transport::HTTP::Response
+
+            def initialize(http_response)
+              super
+            end
+          end
+
+          module API
+            # Endpoint for submitting DSM stats data
+            class Endpoint < Core::Transport::HTTP::API::Endpoint
+              def initialize(path)
+                super(:post, path)
+              end
+
+              def call(env, &block)
+                # Build request
+                env.verb = verb
+                env.path = path
+                env.body = env.request.parcel.data
+
+                # Send request
+                http_response = yield(env)
+
+                # Build response
+                Response.new(http_response)
+              end
+
+              def encoder
+                # DSM handles encoding in the transport layer
+                nil
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/data_streams/transport/http.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require_relative '../../core/transport/http'
+require_relative 'http/stats'
+require_relative 'stats'
+
+module Datadog
+  module DataStreams
+    module Transport
+      # HTTP transport for Data Streams Monitoring
+      module HTTP
+        V01 = Stats::API::Endpoint.new(
+          '/v0.1/pipeline_stats'
+        )
+
+        module_function
+
+        # Builds a new Transport::HTTP::Client with default settings
+        def default(
+          agent_settings:,
+          logger:
+        )
+          Core::Transport::HTTP.build(
+            agent_settings: agent_settings,
+            logger: logger,
+            headers: {
+              'Content-Type' => 'application/msgpack',
+              'Content-Encoding' => 'gzip'
+            }
+          ) do |transport|
+            transport.api 'v0.1', V01, default: true
+
+            # Call block to apply any customization, if provided
+            yield(transport) if block_given?
+          end.to_transport(Transport::Stats::Transport)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/data_streams/transport/stats.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require 'msgpack'
+require 'zlib'
+require_relative '../../core/transport/parcel'
+require_relative '../../core/transport/request'
+require_relative '../../core/transport/transport'
+
+module Datadog
+  module DataStreams
+    module Transport
+      module Stats
+        # Parcel for encoded DSM stats payload
+        class EncodedParcel
+          include Datadog::Core::Transport::Parcel
+
+          def initialize(data)
+            @data = data
+          end
+
+          attr_reader :data
+        end
+
+        # Request for DSM stats
+        class Request < Datadog::Core::Transport::Request
+        end
+
+        # Transport for Data Streams Monitoring stats
+        class Transport < Core::Transport::Transport
+          def send_stats(payload)
+            # MessagePack encode and gzip compress the payload
+            msgpack_data = MessagePack.pack(payload)
+            compressed_data = Zlib.gzip(msgpack_data)
+
+            # Create parcel and request
+            parcel = EncodedParcel.new(compressed_data)
+            request = Request.new(parcel)
+
+            # Send to agent
+            client.send_request(:stats, request)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/data_streams.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+require_relative 'data_streams/processor'
+require_relative 'data_streams/pathway_context'
+require_relative 'data_streams/configuration/settings'
+require_relative 'data_streams/extensions'
+require_relative 'core/utils/time'
+
+module Datadog
+  # Datadog Data Streams Monitoring public API.
+  #
+  # The Datadog team ensures that public methods in this module
+  # only receive backwards compatible changes, and breaking changes
+  # will only occur in new major versions releases.
+  # @public_api
+  module DataStreams
+    class << self
+      # Set a produce checkpoint for Data Streams Monitoring
+      #
+      # @param type [String] The type of the checkpoint (e.g., 'kafka', 'kinesis', 'sns')
+      # @param destination [String] The destination (e.g., topic, exchange, stream name)
+      # @param auto_instrumentation [Boolean] Whether this checkpoint was set by auto-instrumentation (default: false)
+      # @param tags [Hash] Additional tags to include
+      # @yield [key, value] Block to inject context into carrier
+      # @return [String, nil] Base64 encoded pathway context or nil if disabled
+      # @public_api
+      def set_produce_checkpoint(type:, destination:, auto_instrumentation: false, tags: {}, &block)
+        processor&.set_produce_checkpoint(
+          type: type,
+          destination: destination,
+          manual_checkpoint: !auto_instrumentation,
+          tags: tags,
+          &block
+        )
+      end
+
+      # Set a consume checkpoint for Data Streams Monitoring
+      #
+      # @param type [String] The type of the checkpoint (e.g., 'kafka', 'kinesis', 'sns')
+      # @param source [String] The source (e.g., topic, exchange, stream name)
+      # @param auto_instrumentation [Boolean] Whether this checkpoint was set by auto-instrumentation (default: false)
+      # @param tags [Hash] Additional tags to include
+      # @yield [key] Block to extract context from carrier
+      # @return [String, nil] Base64 encoded pathway context or nil if disabled
+      # @public_api
+      def set_consume_checkpoint(type:, source:, auto_instrumentation: false, tags: {}, &block)
+        processor&.set_consume_checkpoint(
+          type: type,
+          source: source,
+          manual_checkpoint: !auto_instrumentation,
+          tags: tags,
+          &block
+        )
+      end
+
+      # Track Kafka produce offset for lag monitoring
+      #
+      # @param topic [String] The Kafka topic name
+      # @param partition [Integer] The partition number
+      # @param offset [Integer] The offset of the produced message
+      # @return [Boolean, nil] true if tracking succeeded, nil if disabled
+      # @!visibility private
+      def track_kafka_produce(topic, partition, offset)
+        processor&.track_kafka_produce(topic, partition, offset, Core::Utils::Time.now)
+      end
+
+      # Track Kafka message consumption for consumer lag monitoring
+      #
+      # @param topic [String] The Kafka topic name
+      # @param partition [Integer] The partition number
+      # @param offset [Integer] The offset of the consumed message
+      # @return [Boolean, nil] true if tracking succeeded, nil if disabled
+      # @!visibility private
+      def track_kafka_consume(topic, partition, offset)
+        processor&.track_kafka_consume(topic, partition, offset, Core::Utils::Time.now)
+      end
+
+      # Check if Data Streams Monitoring is enabled and available
+      #
+      # @return [Boolean] true if the processor is available
+      # @public_api
+      def enabled?
+        !processor.nil?
+      end
+
+      private
+
+      def processor
+        components.data_streams
+      end
+
+      def components
+        Datadog.send(:components)
+      end
+    end
+
+    # Expose Data Streams to global shared objects
+    Extensions.activate!
+  end
+end

data/lib/datadog/di/boot.rb

@@ -17,7 +17,8 @@ require_relative 'serializer'
 require_relative 'transport/http'
 require_relative 'utils'
 
-if %w[1 true yes].include?(Datadog::DATADOG_ENV['DD_DYNAMIC_INSTRUMENTATION_ENABLED']) # steep:ignore
+# Steep: https://github.com/ruby/rbs/pull/2715
+if %w[1 true yes].include?(Datadog::DATADOG_ENV['DD_DYNAMIC_INSTRUMENTATION_ENABLED']) # steep:ignore ArgumentTypeMismatch
 
   # For initial release of Dynamic Instrumentation, activate code tracking
   # only if DI is explicitly requested in the environment.
@@ -35,7 +36,8 @@ require_relative 'contrib'
 
 Datadog::DI::Contrib.load_now_or_later
 
-if %w[1 true yes].include?(Datadog::DATADOG_ENV['DD_DYNAMIC_INSTRUMENTATION_ENABLED']) # steep:ignore
+# Steep: https://github.com/ruby/rbs/pull/2715
+if %w[1 true yes].include?(Datadog::DATADOG_ENV['DD_DYNAMIC_INSTRUMENTATION_ENABLED']) # steep:ignore ArgumentTypeMismatch
   if Datadog::DATADOG_ENV['DD_DYNAMIC_INSTRUMENTATION_PROBE_FILE']
     require_relative 'probe_file_loader'
     Datadog::DI::ProbeFileLoader.load_now_or_later

data/lib/datadog/di/component.rb

@@ -29,22 +29,6 @@ module Datadog
           end
         end
 
-        def build!(settings, agent_settings, logger, telemetry: nil)
-          unless settings.respond_to?(:dynamic_instrumentation) && settings.dynamic_instrumentation.enabled
-            raise "Requested DI component but DI is not enabled in settings"
-          end
-
-          unless settings.respond_to?(:remote) && settings.remote.enabled
-            raise "Requested DI component but remote config is not enabled in settings"
-          end
-
-          unless environment_supported?(settings, logger)
-            raise "DI does not support the environment (development or Ruby version too low or not MRI)"
-          end
-
-          new(settings, agent_settings, logger, code_tracker: DI.code_tracker, telemetry: telemetry)
-        end
-
         # Checks whether the runtime environment is supported by
         # dynamic instrumentation. Currently we only require that, if Rails
         # is used, that Rails environment is not development because

data/lib/datadog/di/contrib/active_record.rb

@@ -1,12 +1,37 @@
 # frozen_string_literal: true
 
-Datadog::DI::Serializer.register(condition: lambda { |value| ActiveRecord::Base === value }) do |serializer, value, name:, depth:| # steep:ignore
-  # steep thinks all of the arguments are nil here
-  # steep:ignore:start
+Datadog::DI::Serializer.register(
+  # This serializer uses a dynamic condition to determine its applicability
+  # to a particular value. A simpler case could have been a serializer for
+  # a particular class, but in this case any ActiveRecord model is covered
+  # and they all have different classes.
+  #
+  # An alternative could have been to make DI specifically provide lookup
+  # logic for "instances of classes derived from X", but a condition Proc
+  # is more universal.
+  condition: lambda { |value| ActiveRecord::Base === value }
+) do |serializer, value, name:, depth:|
+  # +serializer+ is an instance of DI::Serializer.
+  # Use it to perform the serialization to primitive values.
+  #
+  # +value+ is the value to serialize. It should match the condition
+  # provided above, meaning it would be an ActiveRecord::Base instance.
+  #
+  # +name+ is the name of the (local/instance) variable being serialized.
+  # The name is used by DI for redaction (upstream of serialization logic),
+  # and could potentially be used for redaction here also.
+  #
+  # +depth+ is the remaining depth for serializing collections and objects.
+  # It should always be an integer.
+  # Reduce it by 1 when invoking +serialize_value+ on the contents of +value+.
+  # This serializer could also potentially do its own depth limiting.
+  #
+  # Steep: steep thinks all of the arguments are nil here
+  # Looks like it cannot handle kwargs in lambdas
+  # @type var depth: Integer
   value_to_serialize = {
     attributes: value.attributes,
     new_record: value.new_record?,
   }
-  serializer.serialize_value(value_to_serialize, depth: depth ? depth - 1 : nil, type: value.class)
-  # steep:ignore:end
+  serializer.serialize_value(value_to_serialize, depth: depth - 1, type: value.class)
 end

data/lib/datadog/di/el/compiler.rb

@@ -22,7 +22,8 @@ module Datadog
 
         private
 
-        OPERATORS = {
+        # Steep: https://github.com/soutaro/steep/issues/363
+        OPERATORS = { # steep:ignore IncompatibleAssignment
           'eq' => '==',
           'ne' => '!=',
           'ge' => '>=',
@@ -31,16 +32,19 @@ module Datadog
           'lt' => '<',
         }.freeze
 
+        # Steep: https://github.com/soutaro/steep/issues/363
         SINGLE_ARG_METHODS = %w[
           len isEmpty isUndefined
-        ].freeze
+        ].freeze # steep:ignore IncompatibleAssignment
 
+        # Steep: https://github.com/soutaro/steep/issues/363
         TWO_ARG_METHODS = %w[
           startsWith endsWith contains matches
           getmember index instanceof
-        ].freeze
+        ].freeze # steep:ignore IncompatibleAssignment
 
-        MULTI_ARG_METHODS = {
+        # Steep: https://github.com/soutaro/steep/issues/363
+        MULTI_ARG_METHODS = { # steep:ignore IncompatibleAssignment
           'and' => '&&',
           'or' => '||',
         }.freeze

data/lib/datadog/di/el/evaluator.rb

@@ -17,7 +17,7 @@ module Datadog
 
         def len(var, var_name)
           case var
-          when Array, String
+          when Array, String, Hash
             var.length
           else
             raise DI::Error::ExpressionEvaluationError, "Unsupported type for length: #{var.class}: #{var_name}"

data/lib/datadog/di/error.rb

@@ -10,6 +10,10 @@ module Datadog
     #
     # @api private
     class Error < StandardError
+      # Internal Dynamic Instrumentation error ("should never happen").
+      class InternalError < Error
+      end
+
       # Probe does not contain a line number (i.e., is not a line probe).
       class MissingLineNumber < Error
       end
@@ -38,6 +42,11 @@ module Datadog
       class ProbePreviouslyFailed < Error
       end
 
+      # Raised when trying to instrument a probe when there is existing
+      # instrumentation for the same probe id.
+      class AlreadyInstrumented < Error
+      end
+
       # Raised when installing a line probe and multiple files match the
       # specified path suffix.
       # A probe must be installed into one file only, since UI only

data/lib/datadog/di/instrumenter.rb

@@ -89,11 +89,7 @@ module Datadog
       # from the method but from outside of the method).
       Location = Struct.new(:path, :lineno, :label)
 
-      def hook_method(probe, &block)
-        unless block
-          raise ArgumentError, 'block is required'
-        end
-
+      def hook_method(probe, responder)
         lock.synchronize do
           if probe.instrumentation_module
             # Already instrumented, warn?
@@ -118,7 +114,9 @@ module Datadog
         settings = self.settings
 
         mod = Module.new do
-          define_method(method_name) do |*args, **kwargs, &target_block| # steep:ignore
+          define_method(method_name) do |*args, **kwargs, &target_block| # steep:ignore NoMethod
+            # Steep: Unsure why it cannot detect kwargs in this block. Workaround:
+            # @type var kwargs: ::Hash[::Symbol, untyped]
             continue = true
             if condition = probe.condition
               begin
@@ -130,10 +128,34 @@ module Datadog
                   caller_locations: caller_locations,
                 )
                 continue = condition.satisfied?(context)
-              rescue
-                raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
+              rescue => exc
+                # Evaluation error exception can be raised for "expected"
+                # errors, we probably need another setting to control whether
+                # these exceptions are propagated.
+                raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions &&
+                  !exc.is_a?(DI::Error::ExpressionEvaluationError)
+
+                if context
+                  # We want to report evaluation errors for conditions
+                  # as probe snapshots. However, if we failed to create
+                  # the context, we won't be able to report anything as
+                  # the probe notifier builder requires a context.
+                  begin
+                    responder.probe_condition_evaluation_failed_callback(context, exc)
+                  rescue
+                    raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
+
+                    # TODO log / report via telemetry?
+                  end
+                else
+                  _ = 42 # stop standard from wrecking this code
+
+                  raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
+
+                  # TODO log / report via telemetry?
+                  # If execution gets here, there is probably a bug in the tracer.
+                end
 
-                # TODO log / report via telemetry?
                 continue = false
               end
             end
@@ -146,7 +168,10 @@ module Datadog
                   depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
                   attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count)
               end
-              start_time = Core::Utils::Time.get_time
+              # We intentionally do not use Core::Utils::Time.get_time
+              # here because the time provider may be overridden by the
+              # customer, and DI is not allowed to invoke customer code.
+              start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
 
               rv = nil
               begin
@@ -170,7 +195,7 @@ module Datadog
                 # the instrumentation callback runs.
               end
 
-              duration = Core::Utils::Time.get_time - start_time
+              duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
               # The method itself is not part of the stack trace because
               # we are getting the stack trace from outside of the method.
               # Add the method in manually as the top frame.
@@ -186,7 +211,8 @@ module Datadog
                 # that location here.
                 []
               end
-              caller_locs = method_frame + caller_locations # steep:ignore
+              # Steep: https://github.com/ruby/rbs/pull/2745
+              caller_locs = method_frame + caller_locations # steep:ignore ArgumentTypeMismatch
               # TODO capture arguments at exit
 
               context = Context.new(locals: nil, target_self: self,
@@ -195,8 +221,7 @@ module Datadog
                 caller_locations: caller_locs,
                 return_value: rv, duration: duration, exception: exc,)
 
-              # & is to stop steep complaints, block is always present here.
-              block&.call(context)
+              responder.probe_executed_callback(context)
               if exc
                 raise exc
               else
@@ -237,6 +262,8 @@ module Datadog
 
           probe.instrumentation_module = mod
           cls.send(:prepend, mod)
+
+          DI.instrumented_count_inc(:method)
         end
       end
 
@@ -249,6 +276,8 @@ module Datadog
           if mod = probe.instrumentation_module
             mod.send(:remove_method, probe.method_name)
             probe.instrumentation_module = nil
+
+            DI.instrumented_count_dec(:method)
           end
         end
       end
@@ -258,11 +287,7 @@ module Datadog
       # not for eval'd code, unless the eval'd code is associated with
       # a file name and client invokes this method with the correct
       # file name for the eval'd code.
-      def hook_line(probe, &block)
-        unless block
-          raise ArgumentError, 'No block given to hook_line'
-        end
-
+      def hook_line(probe, responder)
         lock.synchronize do
           if probe.instrumentation_trace_point
             # Already instrumented, warn?
@@ -283,7 +308,10 @@ module Datadog
 
         iseq = nil
         if code_tracker
-          ret = code_tracker.iseqs_for_path_suffix(probe.file) # steep:ignore
+          # Steep: Complex type narrowing (before calling hook_line,
+          # we check that probe.line? is true which itself checks that probe.file is not nil)
+          # Annotation do not work here as `file` is a method on probe, not a local variable.
+          ret = code_tracker.iseqs_for_path_suffix(probe.file) # steep:ignore ArgumentTypeMismatch
           unless ret
             if permit_untargeted_trace_points
               # Continue withoout targeting the trace point.
@@ -367,14 +395,44 @@ module Datadog
 
             if continue
               if condition = probe.condition
-                context = Context.new(
-                  locals: Instrumenter.get_local_variables(tp),
-                  target_self: tp.self,
-                  probe: probe, settings: settings, serializer: serializer,
-                  path: tp.path,
-                  caller_locations: caller_locations,
-                )
-                continue = condition.satisfied?(context)
+                begin
+                  context = Context.new(
+                    locals: Instrumenter.get_local_variables(tp),
+                    target_self: tp.self,
+                    probe: probe, settings: settings, serializer: serializer,
+                    path: tp.path,
+                    caller_locations: caller_locations,
+                  )
+                  continue = condition.satisfied?(context)
+                rescue => exc
+                  # Evaluation error exception can be raised for "expected"
+                  # errors, we probably need another setting to control whether
+                  # these exceptions are propagated.
+                  raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions &&
+                    !exc.is_a?(DI::Error::ExpressionEvaluationError)
+
+                  continue = false
+                  if context
+                    # We want to report evaluation errors for conditions
+                    # as probe snapshots. However, if we failed to create
+                    # the context, we won't be able to report anything as
+                    # the probe notifier builder requires a context.
+                    begin
+                      responder.probe_condition_evaluation_failed_callback(context, condition, exc)
+                    rescue
+                      raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
+
+                      # TODO log / report via telemetry?
+                    end
+                  else
+                    _ = 42 # stop standard from wrecking this code
+
+                    raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
+
+                    # TODO log / report via telemetry?
+                    # If execution gets here, there is probably a bug in the tracer.
+                  end
+                end
               end
             end
 
@@ -393,8 +451,7 @@ module Datadog
                 caller_locations: caller_locations,
               )
 
-              # & is to stop steep complaints, block is always present here.
-              block&.call(context)
+              responder.probe_executed_callback(context)
             end
           rescue => exc
             raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
@@ -409,9 +466,11 @@ module Datadog
           # TODO test this path
         end
 
-        # TODO internal check - remove or use a proper exception
+        # Internal sanity check - untargeted trace points create a huge
+        # performance impact, and we absolutely do not want to set them
+        # accidentally.
         if !iseq && !permit_untargeted_trace_points
-          raise "Trying to use an untargeted trace point when user did not permit it"
+          raise Error::InternalError, "Trying to use an untargeted trace point when user did not permit it"
         end
 
         lock.synchronize do
@@ -424,12 +483,16 @@ module Datadog
           # actual_path could be nil if we don't use targeted trace points.
           probe.instrumented_path = actual_path
 
-          if iseq
+          # TracePoint#enable returns false when it succeeds.
+          rv = if iseq
             tp.enable(target: iseq, target_line: line_no)
           else
             tp.enable
           end
-          # TracePoint#enable returns false when it succeeds.
+
+          DI.instrumented_count_inc(:line)
+
+          rv
         end
         true
       end
@@ -439,15 +502,17 @@ module Datadog
           if tp = probe.instrumentation_trace_point
             tp.disable
             probe.instrumentation_trace_point = nil
+
+            DI.instrumented_count_dec(:line)
           end
         end
       end
 
-      def hook(probe, &block)
+      def hook(probe, responder)
         if probe.method?
-          hook_method(probe, &block)
+          hook_method(probe, responder)
         elsif probe.line?
-          hook_line(probe, &block)
+          hook_line(probe, responder)
         else
           # TODO add test coverage for this path
           logger.debug { "di: unknown probe type to hook: #{probe}" }

data/lib/datadog/di/logger.rb

@@ -8,7 +8,7 @@ module Datadog
     #
     # @api private
     class Logger
-      extend Forwardable # steep:ignore
+      extend Forwardable
 
       def initialize(settings, target)
         @settings = settings
@@ -18,7 +18,7 @@ module Datadog
       attr_reader :settings
       attr_reader :target
 
-      def_delegators :target, :debug # steep:ignore
+      def_delegators :target, :debug
 
       def trace(&block)
         if settings.dynamic_instrumentation.internal.trace_logging