datadog 2.12.0 → 2.22.0

data/lib/datadog/core/workers/interval_loop.rb

@@ -21,7 +21,18 @@ module Datadog
         # Methods that must be prepended
         module PrependedMethods
           def perform(*args)
-            perform_loop { super(*args) }
+            perform_loop do
+              @in_iteration = true
+              begin
+                super(*args)
+              ensure
+                @in_iteration = false
+              end
+            end
+          end
+
+          def in_iteration?
+            defined?(@in_iteration) && @in_iteration
           end
         end
 

data/lib/datadog/core/workers/runtime_metrics.rb

@@ -20,8 +20,8 @@ module Datadog
         attr_reader \
           :metrics
 
-        def initialize(options = {})
-          @metrics = options.fetch(:metrics) { Core::Runtime::Metrics.new }
+        def initialize(telemetry:, **options)
+          @metrics = options.fetch(:metrics) { Core::Runtime::Metrics.new(logger: options[:logger], telemetry: telemetry) }
 
           # Workers::Async::Thread settings
           self.fork_policy = options.fetch(:fork_policy, Workers::Async::Thread::FORK_POLICY_STOP)

data/lib/datadog/core.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative 'core/deprecations'
+require_relative 'core/configuration/config_helper'
 require_relative 'core/extensions'
 
 # We must load core extensions to make certain global APIs
@@ -11,8 +12,17 @@ module Datadog
   # for higher-level features.
   module Core
     extend Core::Deprecations
+
+    LIBDATADOG_API_FAILURE =
+      begin
+        require "libdatadog_api.#{RUBY_VERSION[/\d+.\d+/]}_#{RUBY_PLATFORM}"
+        nil
+      rescue LoadError => e
+        e.message
+      end
   end
 
+  DATADOG_ENV = Core::Configuration::ConfigHelper.new
   extend Core::Extensions
 
   # Add shutdown hook:

data/lib/datadog/di/boot.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require_relative 'logger'
+require_relative 'base'
+require_relative 'error'
+require_relative 'code_tracker'
+require_relative 'component'
+require_relative 'context'
+require_relative 'instrumenter'
+require_relative 'probe'
+require_relative 'probe_builder'
+require_relative 'probe_manager'
+require_relative 'probe_notification_builder'
+require_relative 'probe_notifier_worker'
+require_relative 'redactor'
+require_relative 'serializer'
+require_relative 'transport/http'
+require_relative 'utils'
+
+if %w[1 true yes].include?(Datadog::DATADOG_ENV['DD_DYNAMIC_INSTRUMENTATION_ENABLED']) # steep:ignore
+
+  # For initial release of Dynamic Instrumentation, activate code tracking
+  # only if DI is explicitly requested in the environment.
+  # Code tracking is required for line probes to work; see the comments
+  # above for the implementation of the method.
+  #
+  # If DI is enabled programmatically, the application can (and must,
+  # for line probes to work) activate tracking in an initializer.
+  # We seem to have Datadog.logger here already
+  Datadog.logger.debug("di: activating code tracking")
+  Datadog::DI.activate_tracking
+end
+
+require_relative 'contrib'
+
+Datadog::DI::Contrib.load_now_or_later
+
+if %w[1 true yes].include?(Datadog::DATADOG_ENV['DD_DYNAMIC_INSTRUMENTATION_ENABLED']) # steep:ignore
+  if Datadog::DATADOG_ENV['DD_DYNAMIC_INSTRUMENTATION_PROBE_FILE']
+    require_relative 'probe_file_loader'
+    Datadog::DI::ProbeFileLoader.load_now_or_later
+  end
+end

data/lib/datadog/di/component.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require_relative '../core'
-
 module Datadog
   module DI
     # Component for dynamic instrumentation.
@@ -114,6 +112,27 @@ module Datadog
         probe_manager.close
         probe_notifier_worker.stop
       end
+
+      def parse_probe_spec_and_notify(probe_spec)
+        probe = ProbeBuilder.build_from_remote_config(probe_spec)
+      rescue => exc
+        begin
+          probe = Struct.new(:id).new(
+            probe_spec['id'],
+          )
+          payload = probe_notification_builder.build_errored(probe, exc)
+          probe_notifier_worker.add_status(payload)
+        rescue # standard:disable Lint/UselessRescue
+          # TODO report via instrumentation telemetry?
+          raise
+        end
+
+        raise
+      else
+        payload = probe_notification_builder.build_received(probe)
+        probe_notifier_worker.add_status(payload)
+        probe
+      end
     end
   end
 end

data/lib/datadog/di/context.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    # Contains local and instance variables used when evaluating
+    # expressions in DI Expression Language.
+    #
+    # @api private
+    class Context
+      def initialize(probe:, settings:, serializer:, locals: nil,
+        # In Ruby everything is a method, therefore we should always have
+        # a target self. However, if we are not capturing a snapshot,
+        # there is no need to pass in the target self.
+        target_self: nil,
+        path: nil, caller_locations: nil,
+        serialized_entry_args: nil,
+        return_value: nil, duration: nil, exception: nil)
+        @probe = probe
+        @settings = settings
+        @serializer = serializer
+        @locals = locals
+        @target_self = target_self
+        @path = path
+        @caller_locations = caller_locations
+        @serialized_entry_args = serialized_entry_args
+        @return_value = return_value
+        @duration = duration
+        @exception = exception
+      end
+
+      attr_reader :probe
+      attr_reader :settings
+      attr_reader :serializer
+      attr_reader :locals
+      attr_reader :target_self
+      # Actual path of the instrumented file.
+      attr_reader :path
+      # TODO check how many stack frames we should be keeping/sending,
+      # this should be all frames for enriched probes and no frames for
+      # non-enriched probes?
+      attr_reader :caller_locations
+      attr_reader :serialized_entry_args
+      # Return value for the method, for a method probe
+      attr_reader :return_value
+      # How long the method took to execute, for a method probe
+      attr_reader :duration
+      # Exception raised by the method, if any, for a method probe
+      attr_reader :exception
+
+      def serialized_locals
+        # TODO cache?
+        locals && serializer.serialize_vars(locals,
+          depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
+          attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count,)
+      end
+
+      def fetch(var_name)
+        unless locals
+          # TODO return "undefined" instead?
+          return nil
+        end
+        locals[var_name.to_sym]
+      end
+
+      def fetch_ivar(var_name)
+        target_self.instance_variable_get(var_name)
+      end
+    end
+  end
+end

data/lib/datadog/di/el/compiler.rb

@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    module EL
+      # DI Expression Language compiler.
+      #
+      # Converts AST in probe definitions into Expression objects.
+      #
+      # WARNING: this class produces strings that are then eval'd as
+      # Ruby code. Input ASTs are user-controlled. As such the compiler
+      # must sanitize and escape all input to avoid injection.
+      #
+      # Besides quotes and backslashes we must also escape # which is
+      # starting string interpolation (#{...}).
+      #
+      # @api private
+      class Compiler
+        def compile(ast)
+          compile_partial(ast)
+        end
+
+        private
+
+        OPERATORS = {
+          'eq' => '==',
+          'ne' => '!=',
+          'ge' => '>=',
+          'gt' => '>',
+          'le' => '<=',
+          'lt' => '<',
+        }.freeze
+
+        SINGLE_ARG_METHODS = %w[
+          len isEmpty isUndefined
+        ].freeze
+
+        TWO_ARG_METHODS = %w[
+          startsWith endsWith contains matches
+          getmember index instanceof
+        ].freeze
+
+        MULTI_ARG_METHODS = {
+          'and' => '&&',
+          'or' => '||',
+        }.freeze
+
+        def compile_partial(ast)
+          case ast
+          when Hash
+            if ast.length != 1
+              raise DI::Error::InvalidExpression, "Expected hash of length 1: #{ast}"
+            end
+            op, target = ast.first
+            case op
+            when 'ref'
+              unless String === target
+                raise DI::Error::InvalidExpression, "Bad ref value type: #{target.class}: #{target}"
+              end
+              case target
+              when '@it'
+                'current_item'
+              when '@key'
+                'current_key'
+              when '@value'
+                'current_value'
+              when '@return'
+                # For @return, @duration and @exception we shadow
+                # instance variables.
+                "context.return_value"
+              when '@duration'
+                # There is no way to explicitly format the duration.
+                # TODO come up with better formatting?
+                # We could format to a string here but what if customer
+                # has @duration as part of an expression and wants
+                # to retain it as a number?
+                "(context.duration * 1000)"
+              when '@exception'
+                "context.exception"
+              else
+                # Ruby technically allows all kinds of symbols in variable
+                # names, for example spaces and many characters.
+                # Start out with strict validation to avoid possible
+                # surprises and need to escape.
+                unless target =~ %r{\A(@?)([a-zA-Z0-9_]+)\z}
+                  raise DI::Error::BadVariableName, "Bad variable name: #{target}"
+                end
+                method_name = (($1 == '@') ? 'iref' : 'ref')
+                "#{method_name}('#{target}')"
+              end
+            when *SINGLE_ARG_METHODS
+              method_name = op.gsub(/[A-Z]/) { |m| "_#{m.downcase}" }
+              "#{method_name}(#{compile_partial(target)}, '#{var_name_maybe(target)}')"
+            when *TWO_ARG_METHODS
+              method_name = op.gsub(/[A-Z]/) { |m| "_#{m.downcase}" }
+              unless Array === target && target.length == 2
+                raise DI::Error::InvalidExpression, "Improper #{op} syntax"
+              end
+              first, second = target
+              "#{method_name}(#{compile_partial(first)}, (#{compile_partial(second)}))"
+            when *MULTI_ARG_METHODS.keys
+              unless Array === target && target.length >= 1
+                raise DI::Error::InvalidExpression, "Improper #{op} syntax"
+              end
+              compiled_targets = target.map do |item|
+                "(#{compile_partial(item)})"
+              end
+              compiled_op = MULTI_ARG_METHODS[op]
+              "(#{compiled_targets.join(" #{compiled_op} ")})"
+            when 'substring'
+              unless Array === target && target.length == 3
+                raise DI::Error::InvalidExpression, "Improper #{op} syntax"
+              end
+              "#{op}(#{target.map { |arg| "(#{compile_partial(arg)})" }.join(", ")})"
+            when 'not'
+              "!(#{compile_partial(target)})"
+            when *OPERATORS.keys
+              unless Array === target && target.length == 2
+                raise DI::Error::InvalidExpression, "Improper #{op} syntax"
+              end
+              first, second = target
+              operator = OPERATORS.fetch(op)
+              "(#{compile_partial(first)}) #{operator} (#{compile_partial(second)})"
+            when 'any', 'all', 'filter'
+              "#{op}(#{compile_partial(target.first)}) { |current_item, current_key, current_value| #{compile_partial(target.last)} }"
+            else
+              raise DI::Error::InvalidExpression, "Unknown operation: #{op}"
+            end
+          when Numeric, true, false, nil
+            # No escaping is needed for the values here.
+            ast.inspect
+          when String
+            "\"#{escape(ast)}\""
+          when Array
+            # Arrays are commonly used as arguments of operators/methods,
+            # but there are no arrays at the top level in the syntax that
+            # we currently understand. Provide a helpful error message in case
+            # syntax is expanded in the future.
+            raise DI::Error::InvalidExpression, "Array is not valid at its location, do you need to upgrade dd-trace-rb? #{ast}"
+          else
+            raise DI::Error::InvalidExpression, "Unknown type in AST: #{ast}"
+          end
+        end
+
+        # Returns a textual description of +target+ for use in exception
+        # messages. +target+ could be any expression language expression.
+        # WARNING: the result of this method is included in eval'd code,
+        # it must be sanitized to avoid injection.
+        def var_name_maybe(target)
+          if Hash === target && target.length == 1 && target.keys.first == 'ref' &&
+              String === (value = target.values.first)
+            escape(value)
+          else
+            '(expression)'
+          end
+        end
+
+        def escape(needle)
+          needle.gsub("\\") { "\\\\" }.gsub('"') { "\\\"" }.gsub('#') { "\\#" }
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/el/evaluator.rb

@@ -0,0 +1,159 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    module EL
+      # Evaluator for expression language.
+      #
+      # @api private
+      class Evaluator
+        def ref(var)
+          @context.fetch(var)
+        end
+
+        def iref(var)
+          @context.fetch_ivar(var)
+        end
+
+        def len(var, var_name)
+          case var
+          when Array, String
+            var.length
+          else
+            raise DI::Error::ExpressionEvaluationError, "Unsupported type for length: #{var.class}: #{var_name}"
+          end
+        end
+
+        def is_empty(var, var_name)
+          case var
+          when nil, Numeric
+            false
+          when Array, String
+            var.empty?
+          else
+            raise DI::Error::ExpressionEvaluationError, "Unsupported type for isEmpty: #{var.class}: #{var_name}"
+          end
+        end
+
+        def is_undefined(var, var_name)
+          var.nil?
+        end
+
+        def contains(haystack, needle)
+          if String === haystack && String === needle or # standard:disable Style/AndOr
+              Array === haystack
+            haystack.include?(needle)
+          else
+            raise DI::Error::ExpressionEvaluationError, "Invalid arguments for contains: #{haystack}, #{needle}"
+          end
+        end
+
+        def matches(haystack, needle)
+          re = Regexp.compile(needle)
+          !!(haystack =~ re)
+        end
+
+        def getmember(object, field)
+          object.instance_variable_get("@#{field}")
+        end
+
+        def index(array_or_hash, index_or_key)
+          case array_or_hash
+          when Array
+            case index_or_key
+            when Integer
+              array_or_hash[index_or_key]
+            else
+              raise DI::Error::ExpressionEvaluationError, "Invalid index value: #{index_or_key}"
+            end
+          when Hash
+            array_or_hash[index_or_key]
+          else
+            raise DI::Error::ExpressionEvaluationError, "Invalid argument for index: #{array_or_hash}"
+          end
+        end
+
+        def substring(object, from, to)
+          unless String === object
+            raise DI::Error::ExpressionEvaluationError, "Invalid type for substring: #{object}"
+          end
+          object[from...to]
+        end
+
+        def starts_with(haystack, needle)
+          # To guard against running arbitrary customer code, check that
+          # the haystack is a string. This does not help if customer
+          # overrode String#start_with? but at least it's better than nothing.
+          String === haystack && haystack.start_with?(needle)
+        end
+
+        def ends_with(haystack, needle)
+          String === haystack && haystack.end_with?(needle)
+        end
+
+        def all(collection, &block)
+          case collection
+          when Array
+            collection.all? do |item|
+              block.call(item)
+            end
+          when Hash
+            # For hashes, the expression language has both @it and
+            # @key/@value. Manufacture @it from the key and value.
+            collection.all? do |key, value|
+              block.call([key, value], key, value)
+            end
+          else
+            raise DI::Error::ExpressionEvaluationError, "Bad collection type for all: #{collection.class}"
+          end
+        end
+
+        def any(collection, &block)
+          case collection
+          when Array
+            collection.any? do |item|
+              block.call(item)
+            end
+          when Hash
+            collection.any? do |key, value|
+              # For hashes, the expression language has both @it and
+              # @key/@value. Manufacture @it from the key and value.
+              block.call([key, value], key, value)
+            end
+          else
+            raise DI::Error::ExpressionEvaluationError, "Bad collection type for any: #{collection.class}"
+          end
+        end
+
+        def filter(collection, &block)
+          case collection
+          when Array
+            collection.select do |item|
+              block.call(item)
+            end
+          when Hash
+            collection.select do |key, value|
+              block.call([key, value], key, value)
+            end.to_h
+          else
+            raise DI::Error::ExpressionEvaluationError, "Bad collection type for filter: #{collection.class}"
+          end
+        end
+
+        def instanceof(object, cls_name)
+          cls = object.class
+          loop do
+            if cls.name == cls_name
+              return true
+            end
+            if supercls = cls.superclass # standard:disable Lint/AssignmentInCondition
+              cls = supercls
+            else
+              return false
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/el/expression.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    module EL
+      # Represents an Expression Language expression.
+      #
+      # @api private
+      class Expression
+        def initialize(dsl_expr, compiled_expr)
+          unless String === compiled_expr
+            raise ArgumentError, "compiled_expr must be a string"
+          end
+
+          @dsl_expr = dsl_expr
+
+          cls = Class.new(Evaluator)
+          cls.class_exec do
+            eval(<<-RUBY, Object.new.send(:binding), __FILE__, __LINE__ + 1) # standard:disable Security/Eval
+              def evaluate(context)
+                @context = context
+                #{compiled_expr}
+              end
+            RUBY
+          end
+          @evaluator = cls.new
+        end
+
+        attr_reader :dsl_expr
+        attr_reader :evaluator
+
+        def evaluate(context)
+          @evaluator.evaluate(context)
+        end
+
+        def satisfied?(context)
+          !!evaluate(context)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/el.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+require_relative 'el/expression'
+require_relative 'el/compiler'
+require_relative 'el/evaluator'

data/lib/datadog/di/error.rb

@@ -48,6 +48,31 @@ module Datadog
       # and the user will need to make their suffix more precise.
       class MultiplePathsMatch < Error
       end
+
+      # Base class for exceptions arising during expression language AST
+      # compilation into Ruby code.
+      #
+      # Expression language does not specify behavior in all cases,
+      # leaving some choices to the language implementation in the tracers.
+      # It is therefore possible that some technically valid expressions are
+      # prohibited by our implementation.
+      #
+      # It is also possible that the sanitizers/validators prohibit some
+      # esoteric constructs that are technically valid in Ruby,
+      # for example if instance variable name rules are relaxed to allow
+      # arbitrary characters in them as permitted in method names.
+      class InvalidExpression < Error
+      end
+
+      # Variable name with invalid characters in an expression language
+      # expression.
+      class BadVariableName < InvalidExpression
+      end
+
+      # Base class for exceptions arising when evaluating expression language
+      # expressions.
+      class ExpressionEvaluationError < Error
+      end
     end
   end
 end