datadog 2.20.0 → 2.22.0

data/lib/datadog/core/telemetry/emitter.rb

@@ -31,15 +31,15 @@ module Datadog
           seq_id = self.class.sequence.next
           payload = Request.build_payload(event, seq_id, debug: debug?)
           res = @transport.send_telemetry(request_type: event.type, payload: payload)
-          logger.debug { "Telemetry sent for event `#{event.type}` (response code: #{res.code})" }
+          if res.ok?
+            logger.debug { "Telemetry sent for event `#{event.type}`" }
+          else
+            logger.debug { "Failed to send telemetry for event `#{event.type}`: #{res.inspect}" }
+          end
           res
         rescue => e
           logger.debug {
-            "Unable to send telemetry request for event `#{begin
-              event.type
-            rescue
-              "unknown"
-            end}`: #{e}"
+            "Unable to send telemetry request for event `#{event.respond_to?(:type) ? event.type : event.to_s}`: #{e.class}: #{e}"
           }
           Core::Transport::InternalErrorResponse.new(e)
         end

data/lib/datadog/core/telemetry/event/app_endpoints_loaded.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+
+module Datadog
+  module Core
+    module Telemetry
+      module Event
+        # Telemetry event class for sending 'app-endpoints' payload
+        class AppEndpointsLoaded < Base
+          def initialize(endpoints, is_first:)
+            @endpoints = endpoints
+            @is_first = !!is_first
+          end
+
+          def type
+            'app-endpoints'
+          end
+
+          def payload
+            {
+              is_first: @is_first,
+              endpoints: @endpoints
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/event/app_started.rb

@@ -100,7 +100,7 @@ module Datadog
               ),
 
               # Mix of env var, programmatic and default config, so we use unknown
-              conf_value('DD_AGENT_TRANSPORT', agent_transport, seq_id, 'unknown'),
+              conf_value('DD_AGENT_TRANSPORT', agent_transport, seq_id, 'unknown'), # rubocop:disable CustomCops/EnvStringValidationCop
 
               # writer_options is defined as an option that has a Hash value.
               conf_value(
@@ -177,7 +177,7 @@ module Datadog
             list.push(
               conf_value('instrumentation_source', instrumentation_source, seq_id, 'code'),
               conf_value('DD_INJECT_FORCE', Core::Environment::VariableHelpers.env_to_bool('DD_INJECT_FORCE', false), seq_id, 'env_var'),
-              conf_value('DD_INJECTION_ENABLED', ENV['DD_INJECTION_ENABLED'] || '', seq_id, 'env_var'),
+              conf_value('DD_INJECTION_ENABLED', DATADOG_ENV['DD_INJECTION_ENABLED'] || '', seq_id, 'env_var'),
             )
 
             # Add some more custom additional payload values here

data/lib/datadog/core/telemetry/event.rb

@@ -26,6 +26,7 @@ require_relative 'event/base'
 require_relative 'event/app_client_configuration_change'
 require_relative 'event/app_closing'
 require_relative 'event/app_dependencies_loaded'
+require_relative 'event/app_endpoints_loaded'
 require_relative 'event/app_heartbeat'
 require_relative 'event/app_integrations_change'
 require_relative 'event/app_started'

data/lib/datadog/core/transport/response.rb

@@ -34,7 +34,10 @@ module Datadog
         end
 
         def inspect
-          "#{self.class} ok?:#{ok?} unsupported?:#{unsupported?}, " \
+          maybe_code = if respond_to?(:code)
+            " code:#{code}," # steep:ignore
+          end
+          "#{self.class} ok?:#{ok?},#{maybe_code} unsupported?:#{unsupported?}, " \
             "not_found?:#{not_found?}, client_error?:#{client_error?}, " \
             "server_error?:#{server_error?}, internal_error?:#{internal_error?}, " \
             "payload:#{payload}"

data/lib/datadog/core/utils/network.rb

@@ -13,6 +13,7 @@ module Datadog
           true-client-ip
           x-client-ip
           x-forwarded
+          forwarded
           forwarded-for
           x-cluster-client-ip
           fastly-client-ip
@@ -73,6 +74,8 @@ module Datadog
               next unless value
 
               ips = value.split(',')
+              ips = process_forwarded_header_values(ips) if name == 'forwarded'
+
               ips.each do |ip|
                 parsed_ip = ip_to_ipaddr(ip.strip)
 
@@ -83,6 +86,22 @@ module Datadog
             nil
           end
 
+          def process_forwarded_header_values(values)
+            values.each_with_object([]) do |value, acc|
+              value.downcase!
+
+              value.split(';').each do |tuple_str|
+                tuple_str.strip!
+                next unless tuple_str.start_with?('for=')
+
+                tuple_str.delete_prefix!('for=')
+                tuple_str.delete!('"')
+
+                acc << tuple_str
+              end
+            end
+          end
+
           # Returns whether the given value is more likely to be an IPv4 than an IPv6 address.
           #
           # This is done by checking if a dot (`'.'`) character appears before a colon (`':'`) in the value.

data/lib/datadog/core.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative 'core/deprecations'
+require_relative 'core/configuration/config_helper'
 require_relative 'core/extensions'
 
 # We must load core extensions to make certain global APIs
@@ -21,6 +22,7 @@ module Datadog
       end
   end
 
+  DATADOG_ENV = Core::Configuration::ConfigHelper.new
   extend Core::Extensions
 
   # Add shutdown hook:

data/lib/datadog/di/boot.rb

@@ -5,6 +5,7 @@ require_relative 'base'
 require_relative 'error'
 require_relative 'code_tracker'
 require_relative 'component'
+require_relative 'context'
 require_relative 'instrumenter'
 require_relative 'probe'
 require_relative 'probe_builder'
@@ -16,7 +17,8 @@ require_relative 'serializer'
 require_relative 'transport/http'
 require_relative 'utils'
 
-if %w[1 true yes].include?(ENV['DD_DYNAMIC_INSTRUMENTATION_ENABLED']) # steep:ignore
+if %w[1 true yes].include?(Datadog::DATADOG_ENV['DD_DYNAMIC_INSTRUMENTATION_ENABLED']) # steep:ignore
+
   # For initial release of Dynamic Instrumentation, activate code tracking
   # only if DI is explicitly requested in the environment.
   # Code tracking is required for line probes to work; see the comments
@@ -33,8 +35,8 @@ require_relative 'contrib'
 
 Datadog::DI::Contrib.load_now_or_later
 
-if %w[1 true yes].include?(ENV['DD_DYNAMIC_INSTRUMENTATION_ENABLED']) # steep:ignore
-  if ENV['DD_DYNAMIC_INSTRUMENTATION_PROBE_FILE']
+if %w[1 true yes].include?(Datadog::DATADOG_ENV['DD_DYNAMIC_INSTRUMENTATION_ENABLED']) # steep:ignore
+  if Datadog::DATADOG_ENV['DD_DYNAMIC_INSTRUMENTATION_PROBE_FILE']
     require_relative 'probe_file_loader'
     Datadog::DI::ProbeFileLoader.load_now_or_later
   end

data/lib/datadog/di/component.rb

@@ -115,6 +115,20 @@ module Datadog
 
       def parse_probe_spec_and_notify(probe_spec)
         probe = ProbeBuilder.build_from_remote_config(probe_spec)
+      rescue => exc
+        begin
+          probe = Struct.new(:id).new(
+            probe_spec['id'],
+          )
+          payload = probe_notification_builder.build_errored(probe, exc)
+          probe_notifier_worker.add_status(payload)
+        rescue # standard:disable Lint/UselessRescue
+          # TODO report via instrumentation telemetry?
+          raise
+        end
+
+        raise
+      else
         payload = probe_notification_builder.build_received(probe)
         probe_notifier_worker.add_status(payload)
         probe

data/lib/datadog/di/context.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    # Contains local and instance variables used when evaluating
+    # expressions in DI Expression Language.
+    #
+    # @api private
+    class Context
+      def initialize(probe:, settings:, serializer:, locals: nil,
+        # In Ruby everything is a method, therefore we should always have
+        # a target self. However, if we are not capturing a snapshot,
+        # there is no need to pass in the target self.
+        target_self: nil,
+        path: nil, caller_locations: nil,
+        serialized_entry_args: nil,
+        return_value: nil, duration: nil, exception: nil)
+        @probe = probe
+        @settings = settings
+        @serializer = serializer
+        @locals = locals
+        @target_self = target_self
+        @path = path
+        @caller_locations = caller_locations
+        @serialized_entry_args = serialized_entry_args
+        @return_value = return_value
+        @duration = duration
+        @exception = exception
+      end
+
+      attr_reader :probe
+      attr_reader :settings
+      attr_reader :serializer
+      attr_reader :locals
+      attr_reader :target_self
+      # Actual path of the instrumented file.
+      attr_reader :path
+      # TODO check how many stack frames we should be keeping/sending,
+      # this should be all frames for enriched probes and no frames for
+      # non-enriched probes?
+      attr_reader :caller_locations
+      attr_reader :serialized_entry_args
+      # Return value for the method, for a method probe
+      attr_reader :return_value
+      # How long the method took to execute, for a method probe
+      attr_reader :duration
+      # Exception raised by the method, if any, for a method probe
+      attr_reader :exception
+
+      def serialized_locals
+        # TODO cache?
+        locals && serializer.serialize_vars(locals,
+          depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
+          attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count,)
+      end
+
+      def fetch(var_name)
+        unless locals
+          # TODO return "undefined" instead?
+          return nil
+        end
+        locals[var_name.to_sym]
+      end
+
+      def fetch_ivar(var_name)
+        target_self.instance_variable_get(var_name)
+      end
+    end
+  end
+end

data/lib/datadog/di/el/compiler.rb

@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    module EL
+      # DI Expression Language compiler.
+      #
+      # Converts AST in probe definitions into Expression objects.
+      #
+      # WARNING: this class produces strings that are then eval'd as
+      # Ruby code. Input ASTs are user-controlled. As such the compiler
+      # must sanitize and escape all input to avoid injection.
+      #
+      # Besides quotes and backslashes we must also escape # which is
+      # starting string interpolation (#{...}).
+      #
+      # @api private
+      class Compiler
+        def compile(ast)
+          compile_partial(ast)
+        end
+
+        private
+
+        OPERATORS = {
+          'eq' => '==',
+          'ne' => '!=',
+          'ge' => '>=',
+          'gt' => '>',
+          'le' => '<=',
+          'lt' => '<',
+        }.freeze
+
+        SINGLE_ARG_METHODS = %w[
+          len isEmpty isUndefined
+        ].freeze
+
+        TWO_ARG_METHODS = %w[
+          startsWith endsWith contains matches
+          getmember index instanceof
+        ].freeze
+
+        MULTI_ARG_METHODS = {
+          'and' => '&&',
+          'or' => '||',
+        }.freeze
+
+        def compile_partial(ast)
+          case ast
+          when Hash
+            if ast.length != 1
+              raise DI::Error::InvalidExpression, "Expected hash of length 1: #{ast}"
+            end
+            op, target = ast.first
+            case op
+            when 'ref'
+              unless String === target
+                raise DI::Error::InvalidExpression, "Bad ref value type: #{target.class}: #{target}"
+              end
+              case target
+              when '@it'
+                'current_item'
+              when '@key'
+                'current_key'
+              when '@value'
+                'current_value'
+              when '@return'
+                # For @return, @duration and @exception we shadow
+                # instance variables.
+                "context.return_value"
+              when '@duration'
+                # There is no way to explicitly format the duration.
+                # TODO come up with better formatting?
+                # We could format to a string here but what if customer
+                # has @duration as part of an expression and wants
+                # to retain it as a number?
+                "(context.duration * 1000)"
+              when '@exception'
+                "context.exception"
+              else
+                # Ruby technically allows all kinds of symbols in variable
+                # names, for example spaces and many characters.
+                # Start out with strict validation to avoid possible
+                # surprises and need to escape.
+                unless target =~ %r{\A(@?)([a-zA-Z0-9_]+)\z}
+                  raise DI::Error::BadVariableName, "Bad variable name: #{target}"
+                end
+                method_name = (($1 == '@') ? 'iref' : 'ref')
+                "#{method_name}('#{target}')"
+              end
+            when *SINGLE_ARG_METHODS
+              method_name = op.gsub(/[A-Z]/) { |m| "_#{m.downcase}" }
+              "#{method_name}(#{compile_partial(target)}, '#{var_name_maybe(target)}')"
+            when *TWO_ARG_METHODS
+              method_name = op.gsub(/[A-Z]/) { |m| "_#{m.downcase}" }
+              unless Array === target && target.length == 2
+                raise DI::Error::InvalidExpression, "Improper #{op} syntax"
+              end
+              first, second = target
+              "#{method_name}(#{compile_partial(first)}, (#{compile_partial(second)}))"
+            when *MULTI_ARG_METHODS.keys
+              unless Array === target && target.length >= 1
+                raise DI::Error::InvalidExpression, "Improper #{op} syntax"
+              end
+              compiled_targets = target.map do |item|
+                "(#{compile_partial(item)})"
+              end
+              compiled_op = MULTI_ARG_METHODS[op]
+              "(#{compiled_targets.join(" #{compiled_op} ")})"
+            when 'substring'
+              unless Array === target && target.length == 3
+                raise DI::Error::InvalidExpression, "Improper #{op} syntax"
+              end
+              "#{op}(#{target.map { |arg| "(#{compile_partial(arg)})" }.join(", ")})"
+            when 'not'
+              "!(#{compile_partial(target)})"
+            when *OPERATORS.keys
+              unless Array === target && target.length == 2
+                raise DI::Error::InvalidExpression, "Improper #{op} syntax"
+              end
+              first, second = target
+              operator = OPERATORS.fetch(op)
+              "(#{compile_partial(first)}) #{operator} (#{compile_partial(second)})"
+            when 'any', 'all', 'filter'
+              "#{op}(#{compile_partial(target.first)}) { |current_item, current_key, current_value| #{compile_partial(target.last)} }"
+            else
+              raise DI::Error::InvalidExpression, "Unknown operation: #{op}"
+            end
+          when Numeric, true, false, nil
+            # No escaping is needed for the values here.
+            ast.inspect
+          when String
+            "\"#{escape(ast)}\""
+          when Array
+            # Arrays are commonly used as arguments of operators/methods,
+            # but there are no arrays at the top level in the syntax that
+            # we currently understand. Provide a helpful error message in case
+            # syntax is expanded in the future.
+            raise DI::Error::InvalidExpression, "Array is not valid at its location, do you need to upgrade dd-trace-rb? #{ast}"
+          else
+            raise DI::Error::InvalidExpression, "Unknown type in AST: #{ast}"
+          end
+        end
+
+        # Returns a textual description of +target+ for use in exception
+        # messages. +target+ could be any expression language expression.
+        # WARNING: the result of this method is included in eval'd code,
+        # it must be sanitized to avoid injection.
+        def var_name_maybe(target)
+          if Hash === target && target.length == 1 && target.keys.first == 'ref' &&
+              String === (value = target.values.first)
+            escape(value)
+          else
+            '(expression)'
+          end
+        end
+
+        def escape(needle)
+          needle.gsub("\\") { "\\\\" }.gsub('"') { "\\\"" }.gsub('#') { "\\#" }
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/el/evaluator.rb

@@ -0,0 +1,159 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    module EL
+      # Evaluator for expression language.
+      #
+      # @api private
+      class Evaluator
+        def ref(var)
+          @context.fetch(var)
+        end
+
+        def iref(var)
+          @context.fetch_ivar(var)
+        end
+
+        def len(var, var_name)
+          case var
+          when Array, String
+            var.length
+          else
+            raise DI::Error::ExpressionEvaluationError, "Unsupported type for length: #{var.class}: #{var_name}"
+          end
+        end
+
+        def is_empty(var, var_name)
+          case var
+          when nil, Numeric
+            false
+          when Array, String
+            var.empty?
+          else
+            raise DI::Error::ExpressionEvaluationError, "Unsupported type for isEmpty: #{var.class}: #{var_name}"
+          end
+        end
+
+        def is_undefined(var, var_name)
+          var.nil?
+        end
+
+        def contains(haystack, needle)
+          if String === haystack && String === needle or # standard:disable Style/AndOr
+              Array === haystack
+            haystack.include?(needle)
+          else
+            raise DI::Error::ExpressionEvaluationError, "Invalid arguments for contains: #{haystack}, #{needle}"
+          end
+        end
+
+        def matches(haystack, needle)
+          re = Regexp.compile(needle)
+          !!(haystack =~ re)
+        end
+
+        def getmember(object, field)
+          object.instance_variable_get("@#{field}")
+        end
+
+        def index(array_or_hash, index_or_key)
+          case array_or_hash
+          when Array
+            case index_or_key
+            when Integer
+              array_or_hash[index_or_key]
+            else
+              raise DI::Error::ExpressionEvaluationError, "Invalid index value: #{index_or_key}"
+            end
+          when Hash
+            array_or_hash[index_or_key]
+          else
+            raise DI::Error::ExpressionEvaluationError, "Invalid argument for index: #{array_or_hash}"
+          end
+        end
+
+        def substring(object, from, to)
+          unless String === object
+            raise DI::Error::ExpressionEvaluationError, "Invalid type for substring: #{object}"
+          end
+          object[from...to]
+        end
+
+        def starts_with(haystack, needle)
+          # To guard against running arbitrary customer code, check that
+          # the haystack is a string. This does not help if customer
+          # overrode String#start_with? but at least it's better than nothing.
+          String === haystack && haystack.start_with?(needle)
+        end
+
+        def ends_with(haystack, needle)
+          String === haystack && haystack.end_with?(needle)
+        end
+
+        def all(collection, &block)
+          case collection
+          when Array
+            collection.all? do |item|
+              block.call(item)
+            end
+          when Hash
+            # For hashes, the expression language has both @it and
+            # @key/@value. Manufacture @it from the key and value.
+            collection.all? do |key, value|
+              block.call([key, value], key, value)
+            end
+          else
+            raise DI::Error::ExpressionEvaluationError, "Bad collection type for all: #{collection.class}"
+          end
+        end
+
+        def any(collection, &block)
+          case collection
+          when Array
+            collection.any? do |item|
+              block.call(item)
+            end
+          when Hash
+            collection.any? do |key, value|
+              # For hashes, the expression language has both @it and
+              # @key/@value. Manufacture @it from the key and value.
+              block.call([key, value], key, value)
+            end
+          else
+            raise DI::Error::ExpressionEvaluationError, "Bad collection type for any: #{collection.class}"
+          end
+        end
+
+        def filter(collection, &block)
+          case collection
+          when Array
+            collection.select do |item|
+              block.call(item)
+            end
+          when Hash
+            collection.select do |key, value|
+              block.call([key, value], key, value)
+            end.to_h
+          else
+            raise DI::Error::ExpressionEvaluationError, "Bad collection type for filter: #{collection.class}"
+          end
+        end
+
+        def instanceof(object, cls_name)
+          cls = object.class
+          loop do
+            if cls.name == cls_name
+              return true
+            end
+            if supercls = cls.superclass # standard:disable Lint/AssignmentInCondition
+              cls = supercls
+            else
+              return false
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/el/expression.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    module EL
+      # Represents an Expression Language expression.
+      #
+      # @api private
+      class Expression
+        def initialize(dsl_expr, compiled_expr)
+          unless String === compiled_expr
+            raise ArgumentError, "compiled_expr must be a string"
+          end
+
+          @dsl_expr = dsl_expr
+
+          cls = Class.new(Evaluator)
+          cls.class_exec do
+            eval(<<-RUBY, Object.new.send(:binding), __FILE__, __LINE__ + 1) # standard:disable Security/Eval
+              def evaluate(context)
+                @context = context
+                #{compiled_expr}
+              end
+            RUBY
+          end
+          @evaluator = cls.new
+        end
+
+        attr_reader :dsl_expr
+        attr_reader :evaluator
+
+        def evaluate(context)
+          @evaluator.evaluate(context)
+        end
+
+        def satisfied?(context)
+          !!evaluate(context)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/el.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+require_relative 'el/expression'
+require_relative 'el/compiler'
+require_relative 'el/evaluator'

data/lib/datadog/di/error.rb

@@ -48,6 +48,31 @@ module Datadog
       # and the user will need to make their suffix more precise.
       class MultiplePathsMatch < Error
       end
+
+      # Base class for exceptions arising during expression language AST
+      # compilation into Ruby code.
+      #
+      # Expression language does not specify behavior in all cases,
+      # leaving some choices to the language implementation in the tracers.
+      # It is therefore possible that some technically valid expressions are
+      # prohibited by our implementation.
+      #
+      # It is also possible that the sanitizers/validators prohibit some
+      # esoteric constructs that are technically valid in Ruby,
+      # for example if instance variable name rules are relaxed to allow
+      # arbitrary characters in them as permitted in method names.
+      class InvalidExpression < Error
+      end
+
+      # Variable name with invalid characters in an expression language
+      # expression.
+      class BadVariableName < InvalidExpression
+      end
+
+      # Base class for exceptions arising when evaluating expression language
+      # expressions.
+      class ExpressionEvaluationError < Error
+      end
     end
   end
 end