datadog 2.21.0 → 2.23.0

data/lib/datadog/data_streams/transport/stats.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require 'msgpack'
+require 'zlib'
+require_relative '../../core/transport/parcel'
+require_relative '../../core/transport/request'
+
+module Datadog
+  module DataStreams
+    module Transport
+      module Stats
+        # Parcel for encoded DSM stats payload
+        class EncodedParcel
+          include Datadog::Core::Transport::Parcel
+
+          def initialize(data)
+            @data = data
+          end
+
+          attr_reader :data
+        end
+
+        # Request for DSM stats
+        class Request < Datadog::Core::Transport::Request
+        end
+
+        # Transport for Data Streams Monitoring stats
+        class Transport
+          attr_reader :client, :apis, :current_api_id, :logger
+
+          def initialize(apis, default_api, logger:)
+            @apis = apis
+            @logger = logger
+            @default_api = default_api
+            @current_api_id = default_api
+
+            @client = DataStreams::Transport::HTTP::Client.new(current_api, logger: @logger)
+          end
+
+          def send_stats(payload)
+            # MessagePack encode and gzip compress the payload
+            msgpack_data = MessagePack.pack(payload)
+            compressed_data = Zlib.gzip(msgpack_data)
+
+            # Create parcel and request
+            parcel = EncodedParcel.new(compressed_data)
+            request = Request.new(parcel)
+
+            # Send to agent
+            client.send_stats_payload(request)
+          end
+
+          def current_api
+            apis[@current_api_id]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/data_streams.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+require_relative 'data_streams/processor'
+require_relative 'data_streams/pathway_context'
+require_relative 'data_streams/configuration/settings'
+require_relative 'data_streams/extensions'
+require_relative 'core/utils/time'
+
+module Datadog
+  # Datadog Data Streams Monitoring public API.
+  #
+  # The Datadog team ensures that public methods in this module
+  # only receive backwards compatible changes, and breaking changes
+  # will only occur in new major versions releases.
+  # @public_api
+  module DataStreams
+    class << self
+      # Set a produce checkpoint for Data Streams Monitoring
+      #
+      # @param type [String] The type of the checkpoint (e.g., 'kafka', 'kinesis', 'sns')
+      # @param destination [String] The destination (e.g., topic, exchange, stream name)
+      # @param auto_instrumentation [Boolean] Whether this checkpoint was set by auto-instrumentation (default: false)
+      # @param tags [Hash] Additional tags to include
+      # @yield [key, value] Block to inject context into carrier
+      # @return [String, nil] Base64 encoded pathway context or nil if disabled
+      # @public_api
+      def set_produce_checkpoint(type:, destination:, auto_instrumentation: false, tags: {}, &block)
+        processor&.set_produce_checkpoint(
+          type: type,
+          destination: destination,
+          manual_checkpoint: !auto_instrumentation,
+          tags: tags,
+          &block
+        )
+      end
+
+      # Set a consume checkpoint for Data Streams Monitoring
+      #
+      # @param type [String] The type of the checkpoint (e.g., 'kafka', 'kinesis', 'sns')
+      # @param source [String] The source (e.g., topic, exchange, stream name)
+      # @param auto_instrumentation [Boolean] Whether this checkpoint was set by auto-instrumentation (default: false)
+      # @param tags [Hash] Additional tags to include
+      # @yield [key] Block to extract context from carrier
+      # @return [String, nil] Base64 encoded pathway context or nil if disabled
+      # @public_api
+      def set_consume_checkpoint(type:, source:, auto_instrumentation: false, tags: {}, &block)
+        processor&.set_consume_checkpoint(
+          type: type,
+          source: source,
+          manual_checkpoint: !auto_instrumentation,
+          tags: tags,
+          &block
+        )
+      end
+
+      # Track Kafka produce offset for lag monitoring
+      #
+      # @param topic [String] The Kafka topic name
+      # @param partition [Integer] The partition number
+      # @param offset [Integer] The offset of the produced message
+      # @return [Boolean, nil] true if tracking succeeded, nil if disabled
+      # @!visibility private
+      def track_kafka_produce(topic, partition, offset)
+        processor&.track_kafka_produce(topic, partition, offset, Core::Utils::Time.now)
+      end
+
+      # Track Kafka message consumption for consumer lag monitoring
+      #
+      # @param topic [String] The Kafka topic name
+      # @param partition [Integer] The partition number
+      # @param offset [Integer] The offset of the consumed message
+      # @return [Boolean, nil] true if tracking succeeded, nil if disabled
+      # @!visibility private
+      def track_kafka_consume(topic, partition, offset)
+        processor&.track_kafka_consume(topic, partition, offset, Core::Utils::Time.now)
+      end
+
+      # Check if Data Streams Monitoring is enabled and available
+      #
+      # @return [Boolean] true if the processor is available
+      # @public_api
+      def enabled?
+        !processor.nil?
+      end
+
+      private
+
+      def processor
+        components.data_streams
+      end
+
+      def components
+        Datadog.send(:components)
+      end
+    end
+
+    # Expose Data Streams to global shared objects
+    Extensions.activate!
+  end
+end

data/lib/datadog/di/boot.rb

@@ -5,6 +5,7 @@ require_relative 'base'
 require_relative 'error'
 require_relative 'code_tracker'
 require_relative 'component'
+require_relative 'context'
 require_relative 'instrumenter'
 require_relative 'probe'
 require_relative 'probe_builder'

data/lib/datadog/di/component.rb

@@ -29,22 +29,6 @@ module Datadog
           end
         end
 
-        def build!(settings, agent_settings, logger, telemetry: nil)
-          unless settings.respond_to?(:dynamic_instrumentation) && settings.dynamic_instrumentation.enabled
-            raise "Requested DI component but DI is not enabled in settings"
-          end
-
-          unless settings.respond_to?(:remote) && settings.remote.enabled
-            raise "Requested DI component but remote config is not enabled in settings"
-          end
-
-          unless environment_supported?(settings, logger)
-            raise "DI does not support the environment (development or Ruby version too low or not MRI)"
-          end
-
-          new(settings, agent_settings, logger, code_tracker: DI.code_tracker, telemetry: telemetry)
-        end
-
         # Checks whether the runtime environment is supported by
         # dynamic instrumentation. Currently we only require that, if Rails
         # is used, that Rails environment is not development because
@@ -115,6 +99,20 @@ module Datadog
 
       def parse_probe_spec_and_notify(probe_spec)
         probe = ProbeBuilder.build_from_remote_config(probe_spec)
+      rescue => exc
+        begin
+          probe = Struct.new(:id).new(
+            probe_spec['id'],
+          )
+          payload = probe_notification_builder.build_errored(probe, exc)
+          probe_notifier_worker.add_status(payload)
+        rescue # standard:disable Lint/UselessRescue
+          # TODO report via instrumentation telemetry?
+          raise
+        end
+
+        raise
+      else
         payload = probe_notification_builder.build_received(probe)
         probe_notifier_worker.add_status(payload)
         probe

data/lib/datadog/di/context.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    # Contains local and instance variables used when evaluating
+    # expressions in DI Expression Language.
+    #
+    # @api private
+    class Context
+      def initialize(probe:, settings:, serializer:, locals: nil,
+        # In Ruby everything is a method, therefore we should always have
+        # a target self. However, if we are not capturing a snapshot,
+        # there is no need to pass in the target self.
+        target_self: nil,
+        path: nil, caller_locations: nil,
+        serialized_entry_args: nil,
+        return_value: nil, duration: nil, exception: nil)
+        @probe = probe
+        @settings = settings
+        @serializer = serializer
+        @locals = locals
+        @target_self = target_self
+        @path = path
+        @caller_locations = caller_locations
+        @serialized_entry_args = serialized_entry_args
+        @return_value = return_value
+        @duration = duration
+        @exception = exception
+      end
+
+      attr_reader :probe
+      attr_reader :settings
+      attr_reader :serializer
+      attr_reader :locals
+      attr_reader :target_self
+      # Actual path of the instrumented file.
+      attr_reader :path
+      # TODO check how many stack frames we should be keeping/sending,
+      # this should be all frames for enriched probes and no frames for
+      # non-enriched probes?
+      attr_reader :caller_locations
+      attr_reader :serialized_entry_args
+      # Return value for the method, for a method probe
+      attr_reader :return_value
+      # How long the method took to execute, for a method probe
+      attr_reader :duration
+      # Exception raised by the method, if any, for a method probe
+      attr_reader :exception
+
+      def serialized_locals
+        # TODO cache?
+        locals && serializer.serialize_vars(locals,
+          depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
+          attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count,)
+      end
+
+      def fetch(var_name)
+        unless locals
+          # TODO return "undefined" instead?
+          return nil
+        end
+        locals[var_name.to_sym]
+      end
+
+      def fetch_ivar(var_name)
+        target_self.instance_variable_get(var_name)
+      end
+    end
+  end
+end

data/lib/datadog/di/el/compiler.rb

@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    module EL
+      # DI Expression Language compiler.
+      #
+      # Converts AST in probe definitions into Expression objects.
+      #
+      # WARNING: this class produces strings that are then eval'd as
+      # Ruby code. Input ASTs are user-controlled. As such the compiler
+      # must sanitize and escape all input to avoid injection.
+      #
+      # Besides quotes and backslashes we must also escape # which is
+      # starting string interpolation (#{...}).
+      #
+      # @api private
+      class Compiler
+        def compile(ast)
+          compile_partial(ast)
+        end
+
+        private
+
+        OPERATORS = {
+          'eq' => '==',
+          'ne' => '!=',
+          'ge' => '>=',
+          'gt' => '>',
+          'le' => '<=',
+          'lt' => '<',
+        }.freeze
+
+        SINGLE_ARG_METHODS = %w[
+          len isEmpty isUndefined
+        ].freeze
+
+        TWO_ARG_METHODS = %w[
+          startsWith endsWith contains matches
+          getmember index instanceof
+        ].freeze
+
+        MULTI_ARG_METHODS = {
+          'and' => '&&',
+          'or' => '||',
+        }.freeze
+
+        def compile_partial(ast)
+          case ast
+          when Hash
+            if ast.length != 1
+              raise DI::Error::InvalidExpression, "Expected hash of length 1: #{ast}"
+            end
+            op, target = ast.first
+            case op
+            when 'ref'
+              unless String === target
+                raise DI::Error::InvalidExpression, "Bad ref value type: #{target.class}: #{target}"
+              end
+              case target
+              when '@it'
+                'current_item'
+              when '@key'
+                'current_key'
+              when '@value'
+                'current_value'
+              when '@return'
+                # For @return, @duration and @exception we shadow
+                # instance variables.
+                "context.return_value"
+              when '@duration'
+                # There is no way to explicitly format the duration.
+                # TODO come up with better formatting?
+                # We could format to a string here but what if customer
+                # has @duration as part of an expression and wants
+                # to retain it as a number?
+                "(context.duration * 1000)"
+              when '@exception'
+                "context.exception"
+              else
+                # Ruby technically allows all kinds of symbols in variable
+                # names, for example spaces and many characters.
+                # Start out with strict validation to avoid possible
+                # surprises and need to escape.
+                unless target =~ %r{\A(@?)([a-zA-Z0-9_]+)\z}
+                  raise DI::Error::BadVariableName, "Bad variable name: #{target}"
+                end
+                method_name = (($1 == '@') ? 'iref' : 'ref')
+                "#{method_name}('#{target}')"
+              end
+            when *SINGLE_ARG_METHODS
+              method_name = op.gsub(/[A-Z]/) { |m| "_#{m.downcase}" }
+              "#{method_name}(#{compile_partial(target)}, '#{var_name_maybe(target)}')"
+            when *TWO_ARG_METHODS
+              method_name = op.gsub(/[A-Z]/) { |m| "_#{m.downcase}" }
+              unless Array === target && target.length == 2
+                raise DI::Error::InvalidExpression, "Improper #{op} syntax"
+              end
+              first, second = target
+              "#{method_name}(#{compile_partial(first)}, (#{compile_partial(second)}))"
+            when *MULTI_ARG_METHODS.keys
+              unless Array === target && target.length >= 1
+                raise DI::Error::InvalidExpression, "Improper #{op} syntax"
+              end
+              compiled_targets = target.map do |item|
+                "(#{compile_partial(item)})"
+              end
+              compiled_op = MULTI_ARG_METHODS[op]
+              "(#{compiled_targets.join(" #{compiled_op} ")})"
+            when 'substring'
+              unless Array === target && target.length == 3
+                raise DI::Error::InvalidExpression, "Improper #{op} syntax"
+              end
+              "#{op}(#{target.map { |arg| "(#{compile_partial(arg)})" }.join(", ")})"
+            when 'not'
+              "!(#{compile_partial(target)})"
+            when *OPERATORS.keys
+              unless Array === target && target.length == 2
+                raise DI::Error::InvalidExpression, "Improper #{op} syntax"
+              end
+              first, second = target
+              operator = OPERATORS.fetch(op)
+              "(#{compile_partial(first)}) #{operator} (#{compile_partial(second)})"
+            when 'any', 'all', 'filter'
+              "#{op}(#{compile_partial(target.first)}) { |current_item, current_key, current_value| #{compile_partial(target.last)} }"
+            else
+              raise DI::Error::InvalidExpression, "Unknown operation: #{op}"
+            end
+          when Numeric, true, false, nil
+            # No escaping is needed for the values here.
+            ast.inspect
+          when String
+            "\"#{escape(ast)}\""
+          when Array
+            # Arrays are commonly used as arguments of operators/methods,
+            # but there are no arrays at the top level in the syntax that
+            # we currently understand. Provide a helpful error message in case
+            # syntax is expanded in the future.
+            raise DI::Error::InvalidExpression, "Array is not valid at its location, do you need to upgrade dd-trace-rb? #{ast}"
+          else
+            raise DI::Error::InvalidExpression, "Unknown type in AST: #{ast}"
+          end
+        end
+
+        # Returns a textual description of +target+ for use in exception
+        # messages. +target+ could be any expression language expression.
+        # WARNING: the result of this method is included in eval'd code,
+        # it must be sanitized to avoid injection.
+        def var_name_maybe(target)
+          if Hash === target && target.length == 1 && target.keys.first == 'ref' &&
+              String === (value = target.values.first)
+            escape(value)
+          else
+            '(expression)'
+          end
+        end
+
+        def escape(needle)
+          needle.gsub("\\") { "\\\\" }.gsub('"') { "\\\"" }.gsub('#') { "\\#" }
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/el/evaluator.rb

@@ -0,0 +1,159 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    module EL
+      # Evaluator for expression language.
+      #
+      # @api private
+      class Evaluator
+        def ref(var)
+          @context.fetch(var)
+        end
+
+        def iref(var)
+          @context.fetch_ivar(var)
+        end
+
+        def len(var, var_name)
+          case var
+          when Array, String, Hash
+            var.length
+          else
+            raise DI::Error::ExpressionEvaluationError, "Unsupported type for length: #{var.class}: #{var_name}"
+          end
+        end
+
+        def is_empty(var, var_name)
+          case var
+          when nil, Numeric
+            false
+          when Array, String
+            var.empty?
+          else
+            raise DI::Error::ExpressionEvaluationError, "Unsupported type for isEmpty: #{var.class}: #{var_name}"
+          end
+        end
+
+        def is_undefined(var, var_name)
+          var.nil?
+        end
+
+        def contains(haystack, needle)
+          if String === haystack && String === needle or # standard:disable Style/AndOr
+              Array === haystack
+            haystack.include?(needle)
+          else
+            raise DI::Error::ExpressionEvaluationError, "Invalid arguments for contains: #{haystack}, #{needle}"
+          end
+        end
+
+        def matches(haystack, needle)
+          re = Regexp.compile(needle)
+          !!(haystack =~ re)
+        end
+
+        def getmember(object, field)
+          object.instance_variable_get("@#{field}")
+        end
+
+        def index(array_or_hash, index_or_key)
+          case array_or_hash
+          when Array
+            case index_or_key
+            when Integer
+              array_or_hash[index_or_key]
+            else
+              raise DI::Error::ExpressionEvaluationError, "Invalid index value: #{index_or_key}"
+            end
+          when Hash
+            array_or_hash[index_or_key]
+          else
+            raise DI::Error::ExpressionEvaluationError, "Invalid argument for index: #{array_or_hash}"
+          end
+        end
+
+        def substring(object, from, to)
+          unless String === object
+            raise DI::Error::ExpressionEvaluationError, "Invalid type for substring: #{object}"
+          end
+          object[from...to]
+        end
+
+        def starts_with(haystack, needle)
+          # To guard against running arbitrary customer code, check that
+          # the haystack is a string. This does not help if customer
+          # overrode String#start_with? but at least it's better than nothing.
+          String === haystack && haystack.start_with?(needle)
+        end
+
+        def ends_with(haystack, needle)
+          String === haystack && haystack.end_with?(needle)
+        end
+
+        def all(collection, &block)
+          case collection
+          when Array
+            collection.all? do |item|
+              block.call(item)
+            end
+          when Hash
+            # For hashes, the expression language has both @it and
+            # @key/@value. Manufacture @it from the key and value.
+            collection.all? do |key, value|
+              block.call([key, value], key, value)
+            end
+          else
+            raise DI::Error::ExpressionEvaluationError, "Bad collection type for all: #{collection.class}"
+          end
+        end
+
+        def any(collection, &block)
+          case collection
+          when Array
+            collection.any? do |item|
+              block.call(item)
+            end
+          when Hash
+            collection.any? do |key, value|
+              # For hashes, the expression language has both @it and
+              # @key/@value. Manufacture @it from the key and value.
+              block.call([key, value], key, value)
+            end
+          else
+            raise DI::Error::ExpressionEvaluationError, "Bad collection type for any: #{collection.class}"
+          end
+        end
+
+        def filter(collection, &block)
+          case collection
+          when Array
+            collection.select do |item|
+              block.call(item)
+            end
+          when Hash
+            collection.select do |key, value|
+              block.call([key, value], key, value)
+            end.to_h
+          else
+            raise DI::Error::ExpressionEvaluationError, "Bad collection type for filter: #{collection.class}"
+          end
+        end
+
+        def instanceof(object, cls_name)
+          cls = object.class
+          loop do
+            if cls.name == cls_name
+              return true
+            end
+            if supercls = cls.superclass # standard:disable Lint/AssignmentInCondition
+              cls = supercls
+            else
+              return false
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/el/expression.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    module EL
+      # Represents an Expression Language expression.
+      #
+      # @api private
+      class Expression
+        def initialize(dsl_expr, compiled_expr)
+          unless String === compiled_expr
+            raise ArgumentError, "compiled_expr must be a string"
+          end
+
+          @dsl_expr = dsl_expr
+
+          cls = Class.new(Evaluator)
+          cls.class_exec do
+            eval(<<-RUBY, Object.new.send(:binding), __FILE__, __LINE__ + 1) # standard:disable Security/Eval
+              def evaluate(context)
+                @context = context
+                #{compiled_expr}
+              end
+            RUBY
+          end
+          @evaluator = cls.new
+        end
+
+        attr_reader :dsl_expr
+        attr_reader :evaluator
+
+        def evaluate(context)
+          @evaluator.evaluate(context)
+        end
+
+        def satisfied?(context)
+          !!evaluate(context)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/el.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+require_relative 'el/expression'
+require_relative 'el/compiler'
+require_relative 'el/evaluator'