datadog 2.2.0 → 2.3.0

data/ext/libdatadog_extconf_helpers.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+require 'rubygems'
+require 'pathname'
+
+module Datadog
+  # Contains a bunch of shared helpers that get used during building of extensions that link to libdatadog
+  module LibdatadogExtconfHelpers
+    # Used to make sure the correct gem version gets loaded, as extconf.rb does not get run with "bundle exec" and thus
+    # may see multiple libdatadog versions. See https://github.com/DataDog/dd-trace-rb/pull/2531 for the horror story.
+    LIBDATADOG_VERSION = '~> 11.0.0.1.0'
+
+    # Used as an workaround for a limitation with how dynamic linking works in environments where the datadog gem and
+    # libdatadog are moved after the extension gets compiled.
+    #
+    # Because the libdatadog native library is installed on a non-standard system path, in order for it to be
+    # found by the system dynamic linker (e.g. what takes care of dlopen(), which is used to load
+    # native extensions), we need to add a "runpath" -- a list of folders to search for libdatadog.
+    #
+    # This runpath gets hardcoded at native library linking time. You can look at it using the `readelf` tool in
+    # Linux: e.g. `readelf -d datadog_profiling_native_extension.2.7.3_x86_64-linux.so`.
+    #
+    # In older versions of the datadog gem, we only set as runpath an absolute path to libdatadog.
+    # (This gets set automatically by the call
+    # to `pkg_config('datadog_profiling_with_rpath')` in `extconf.rb`). This worked fine as long as libdatadog was **NOT**
+    # moved from the folder it was present at datadog gem installation/linking time.
+    #
+    # Unfortunately, environments such as Heroku and AWS Elastic Beanstalk move gems around in the filesystem after
+    # installation. Thus, the profiling native extension could not be loaded in these environments
+    # (see https://github.com/DataDog/dd-trace-rb/issues/2067) because libdatadog could not be found.
+    #
+    # To workaround this issue, this method computes the **relative** path between the folder where
+    # native extensions are going to be installed and the folder where libdatadog is installed, and returns it
+    # to be set as an additional runpath. (Yes, you can set multiple runpath folders to be searched).
+    #
+    # This way, if both gems are moved together (and it turns out that they are in these environments),
+    # the relative path can still be traversed to find libdatadog.
+    #
+    # This is incredibly awful, and it's kinda bizarre how it's not possible to just find these paths at runtime
+    # and set them correctly; rather than needing to set stuff at linking-time and then praying to $deity that
+    # weird moves don't happen.
+    #
+    # As a curiosity, `LD_LIBRARY_PATH` can be used to influence the folders that get searched but **CANNOT BE
+    # SET DYNAMICALLY**, e.g. it needs to be set at the start of the process (Ruby VM) and thus it's not something
+    # we could setup when doing a `require`.
+    #
+    def self.libdatadog_folder_relative_to_native_lib_folder(
+      current_folder:,
+      libdatadog_pkgconfig_folder: Libdatadog.pkgconfig_folder
+    )
+      return unless libdatadog_pkgconfig_folder
+
+      native_lib_folder = "#{current_folder}/../../lib/"
+      libdatadog_lib_folder = "#{libdatadog_pkgconfig_folder}/../"
+
+      Pathname.new(libdatadog_lib_folder).relative_path_from(Pathname.new(native_lib_folder)).to_s
+    end
+
+    # In https://github.com/DataDog/dd-trace-rb/pull/3582 we got a report of a customer for which the native extension
+    # only got installed into the extensions folder.
+    #
+    # But then this fix was not enough to fully get them moving because then they started to see the issue from
+    # https://github.com/DataDog/dd-trace-rb/issues/2067 / https://github.com/DataDog/dd-trace-rb/pull/2125 :
+    #
+    # > Profiling was requested but is not supported, profiling disabled: There was an error loading the profiling
+    # > native extension due to 'RuntimeError Failure to load datadog_profiling_native_extension.3.2.2_x86_64-linux
+    # > due to libdatadog_profiling.so: cannot open shared object file: No such file or directory
+    #
+    # The problem is that when loading the native extension from the extensions directory, the relative rpath we add
+    # with the #libdatadog_folder_relative_to_native_lib_folder helper above is not correct, we need to add a relative
+    # rpath to the extensions directory.
+    #
+    # So how do we find the full path where the native extension is placed?
+    # * From https://github.com/ruby/ruby/blob/83f02d42e0a3c39661dc99c049ab9a70ff227d5b/lib/bundler/runtime.rb#L166
+    #   `extension_dirs = Dir["#{Gem.dir}/extensions/*/*/*"] + Dir["#{Gem.dir}/bundler/gems/extensions/*/*/*"]`
+    #   we get that's in one of two fixed subdirectories of `Gem.dir`
+    # * From https://github.com/ruby/ruby/blob/83f02d42e0a3c39661dc99c049ab9a70ff227d5b/lib/rubygems/basic_specification.rb#L111-L115
+    #   we get the structure of the subdirectory (platform/extension_api_version/gem_and_version)
+    #
+    # Thus, `Gem.dir` of `/var/app/current/vendor/bundle/ruby/3.2.0` becomes (for instance)
+    # `/var/app/current/vendor/bundle/ruby/3.2.0/extensions/x86_64-linux/3.2.0/datadog-2.0.0/` or
+    # `/var/app/current/vendor/bundle/ruby/3.2.0/bundler/gems/extensions/x86_64-linux/3.2.0/datadog-2.0.0/`
+    #
+    # We then compute the relative path between these folders and the libdatadog folder, and use that as a relative path.
+    def self.libdatadog_folder_relative_to_ruby_extensions_folders(
+      gem_dir: Gem.dir,
+      libdatadog_pkgconfig_folder: Libdatadog.pkgconfig_folder
+    )
+      return unless libdatadog_pkgconfig_folder
+
+      # For the purposes of calculating a folder relative to the other, we don't actually NEED to fill in the
+      # platform, extension_api_version and gem version. We're basically just after how many folders it is deep from
+      # the Gem.dir.
+      expected_ruby_extensions_folders = [
+        "#{gem_dir}/extensions/platform/extension_api_version/datadog_version/",
+        "#{gem_dir}/bundler/gems/extensions/platform/extension_api_version/datadog_version/",
+      ]
+      libdatadog_lib_folder = "#{libdatadog_pkgconfig_folder}/../"
+
+      expected_ruby_extensions_folders.map do |folder|
+        Pathname.new(libdatadog_lib_folder).relative_path_from(Pathname.new(folder)).to_s
+      end
+    end
+
+    # mkmf sets $PKGCONFIG after the `pkg_config` gets used in extconf.rb. When `pkg_config` is unsuccessful, we use
+    # this helper to decide if we can show more specific error message vs a generic "something went wrong".
+    def self.pkg_config_missing?(command: $PKGCONFIG) # rubocop:disable Style/GlobalVars
+      pkg_config_available = command && xsystem("#{command} --version")
+
+      pkg_config_available != true
+    end
+
+    def self.try_loading_libdatadog
+      gem 'libdatadog', LIBDATADOG_VERSION
+      require 'libdatadog'
+      nil
+    rescue Exception => e # rubocop:disable Lint/RescueException
+      if block_given?
+        yield e
+      else
+        e
+      end
+    end
+
+    def self.libdatadog_issue?
+      try_loading_libdatadog { |_exception| return true }
+      Libdatadog.pkgconfig_folder.nil?
+    end
+  end
+end

data/lib/datadog/appsec/contrib/graphql/appsec_trace.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require 'json'
+require_relative 'gateway/multiplex'
+require_relative '../../instrumentation/gateway'
+
+module Datadog
+  module AppSec
+    module Contrib
+      module GraphQL
+        # These methods will be called by the GraphQL runtime to send the variables to the WAF.
+        # We actually don't need to create any span/trace.
+        module AppSecTrace
+          def execute_multiplex(multiplex:)
+            return super unless Datadog::AppSec.enabled?
+
+            gateway_multiplex = Gateway::Multiplex.new(multiplex)
+
+            multiplex_return, multiplex_response = Instrumentation.gateway.push('graphql.multiplex', gateway_multiplex) do
+              super
+            end
+
+            # Returns an error * the number of queries so that the entire multiplex is blocked
+            if multiplex_response
+              blocked_event = multiplex_response.find { |action, _options| action == :block }
+              multiplex_return = AppSec::Response.graphql_response(gateway_multiplex) if blocked_event
+            end
+
+            multiplex_return
+          end
+
+          private
+
+          def active_trace
+            return unless defined?(Datadog::Tracing)
+
+            Datadog::Tracing.active_trace
+          end
+
+          def active_span
+            return unless defined?(Datadog::Tracing)
+
+            Datadog::Tracing.active_span
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/graphql/gateway/multiplex.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require 'graphql'
+
+require_relative '../../../instrumentation/gateway/argument'
+
+module Datadog
+  module AppSec
+    module Contrib
+      module GraphQL
+        module Gateway
+          # Gateway Request argument. Normalized extration of data from Rack::Request
+          class Multiplex < Instrumentation::Gateway::Argument
+            attr_reader :multiplex
+
+            def initialize(multiplex)
+              super()
+              @multiplex = multiplex
+            end
+
+            def arguments
+              @arguments ||= create_arguments_hash
+            end
+
+            def queries
+              @multiplex.queries
+            end
+
+            private
+
+            def create_arguments_hash
+              args = {}
+              @multiplex.queries.each_with_index do |query, index|
+                resolver_args = {}
+                resolver_dirs = {}
+                selections = (query.selected_operation.selections.dup if query.selected_operation) || []
+                # Iterative tree traversal
+                while selections.any?
+                  selection = selections.shift
+                  set_hash_with_variables(resolver_args, selection.arguments, query.provided_variables)
+                  selection.directives.each do |dir|
+                    resolver_dirs[dir.name] ||= {}
+                    set_hash_with_variables(resolver_dirs[dir.name], dir.arguments, query.provided_variables)
+                  end
+                  selections.concat(selection.selections)
+                end
+                next if resolver_args.empty? && resolver_dirs.empty?
+
+                args_resolver = (args[query.operation_name || "query#{index + 1}"] ||= [])
+                # We don't want to add empty hashes so we check again if the arguments and directives are empty
+                args_resolver << resolver_args unless resolver_args.empty?
+                args_resolver << resolver_dirs unless resolver_dirs.empty?
+              end
+              args
+            end
+
+            # Set the resolver hash (resolver_args and resolver_dirs) with the arguments and provided variables
+            def set_hash_with_variables(resolver_hash, arguments, provided_variables)
+              arguments.each do |arg|
+                resolver_hash[arg.name] =
+                  if arg.value.is_a?(::GraphQL::Language::Nodes::VariableIdentifier)
+                    provided_variables[arg.value.name]
+                  else
+                    arg.value
+                  end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/graphql/gateway/watcher.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require 'json'
+require_relative '../../../instrumentation/gateway'
+require_relative '../reactive/multiplex'
+require_relative '../../../reactive/operation'
+
+module Datadog
+  module AppSec
+    module Contrib
+      module GraphQL
+        module Gateway
+          # Watcher for Rack gateway events
+          module Watcher
+            class << self
+              def watch
+                gateway = Instrumentation.gateway
+
+                watch_multiplex(gateway)
+              end
+
+              # This time we don't throw but use next
+              def watch_multiplex(gateway = Instrumentation.gateway)
+                gateway.watch('graphql.multiplex', :appsec) do |stack, gateway_multiplex|
+                  block = false
+                  event = nil
+                  scope = AppSec::Scope.active_scope
+
+                  AppSec::Reactive::Operation.new('graphql.multiplex') do |op|
+                    GraphQL::Reactive::Multiplex.subscribe(op, scope.processor_context) do |result|
+                      event = {
+                        waf_result: result,
+                        trace: scope.trace,
+                        span: scope.service_entry_span,
+                        multiplex: gateway_multiplex,
+                        actions: result.actions
+                      }
+
+                      if scope.service_entry_span
+                        scope.service_entry_span.set_tag('appsec.blocked', 'true') if result.actions.include?('block')
+                        scope.service_entry_span.set_tag('appsec.event', 'true')
+                      end
+
+                      scope.processor_context.events << event
+                    end
+
+                    block = GraphQL::Reactive::Multiplex.publish(op, gateway_multiplex)
+                  end
+
+                  next [nil, [[:block, event]]] if block
+
+                  ret, res = stack.call(gateway_multiplex.arguments)
+
+                  if event
+                    res ||= []
+                    res << [:monitor, event]
+                  end
+
+                  [ret, res]
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/graphql/integration.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require_relative '../integration'
+require_relative 'patcher'
+
+module Datadog
+  module AppSec
+    module Contrib
+      module GraphQL
+        # Description of GraphQL integration
+        class Integration
+          include Datadog::AppSec::Contrib::Integration
+
+          MINIMUM_VERSION = Gem::Version.new('2.0.19')
+
+          register_as :graphql, auto_patch: false
+
+          def self.version
+            Gem.loaded_specs['graphql'] && Gem.loaded_specs['graphql'].version
+          end
+
+          def self.loaded?
+            !defined?(::GraphQL).nil?
+          end
+
+          def self.compatible?
+            super && version >= MINIMUM_VERSION
+          end
+
+          def self.auto_instrument?
+            true
+          end
+
+          def patcher
+            Patcher
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/graphql/patcher.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require_relative '../patcher'
+require_relative 'gateway/watcher'
+
+if Gem.loaded_specs['graphql'] && Gem.loaded_specs['graphql'].version >= Gem::Version.new('2.0.19')
+  require_relative 'appsec_trace'
+end
+
+module Datadog
+  module AppSec
+    module Contrib
+      module GraphQL
+        # Patcher for AppSec on GraphQL
+        module Patcher
+          include Datadog::AppSec::Contrib::Patcher
+
+          module_function
+
+          def patched?
+            Patcher.instance_variable_get(:@patched)
+          end
+
+          def target_version
+            Integration.version
+          end
+
+          def patch
+            Gateway::Watcher.watch
+            ::GraphQL::Schema.trace_with(AppSecTrace)
+            Patcher.instance_variable_set(:@patched, true)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/graphql/reactive/multiplex.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Contrib
+      module GraphQL
+        module Reactive
+          # Dispatch data from a GraphQL resolve query to the WAF context
+          module Multiplex
+            ADDRESSES = [
+              'graphql.server.all_resolvers'
+            ].freeze
+            private_constant :ADDRESSES
+
+            def self.publish(op, gateway_multiplex)
+              catch(:block) do
+                op.publish('graphql.server.all_resolvers', gateway_multiplex.arguments)
+
+                nil
+              end
+            end
+
+            def self.subscribe(op, waf_context)
+              op.subscribe(*ADDRESSES) do |*values|
+                Datadog.logger.debug { "reacted to #{ADDRESSES.inspect}: #{values.inspect}" }
+                arguments = values[0]
+
+                waf_args = {
+                  'graphql.server.all_resolvers' => arguments
+                }
+
+                waf_timeout = Datadog.configuration.appsec.waf_timeout
+                result = waf_context.run(waf_args, waf_timeout)
+
+                Datadog.logger.debug { "WAF TIMEOUT: #{result.inspect}" } if result.timeout
+
+                case result.status
+                when :match
+                  Datadog.logger.debug { "WAF: #{result.inspect}" }
+
+                  yield result
+                  throw(:block, true) unless result.actions.empty?
+                when :ok
+                  Datadog.logger.debug { "WAF OK: #{result.inspect}" }
+                when :invalid_call
+                  Datadog.logger.debug { "WAF CALL ERROR: #{result.inspect}" }
+                when :invalid_rule, :invalid_flow, :no_rule
+                  Datadog.logger.debug { "WAF RULE ERROR: #{result.inspect}" }
+                else
+                  Datadog.logger.debug { "WAF UNKNOWN: #{result.status.inspect} #{result.inspect}" }
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/rack/gateway/request.rb

@@ -41,7 +41,7 @@ module Datadog
 
             def headers
               result = request.env.each_with_object({}) do |(k, v), h|
-                h[k.gsub(/^HTTP_/, '').downcase!.tr('_', '-')] = v if k =~ /^HTTP_/
+                h[k.delete_prefix('HTTP_').tap(&:downcase!).tap { |s| s.tr!('_', '-') }] = v if k.start_with?('HTTP_')
               end
 
               result['content-type'] = request.content_type if request.content_type

data/lib/datadog/appsec/processor/actions.rb

@@ -11,7 +11,7 @@ module Datadog
             @actions ||= []
           end
 
-          def fecth_configuration(action)
+          def fetch_configuration(action)
             actions.find { |action_configuration| action_configuration['id'] == action }
           end
 

data/lib/datadog/appsec/response.rb

@@ -36,7 +36,7 @@ module Datadog
             # I rather use break to stop the execution
             next if configured_response
 
-            action_configuration = AppSec::Processor::Actions.fecth_configuration(action)
+            action_configuration = AppSec::Processor::Actions.fetch_configuration(action)
             next unless action_configuration
 
             configured_response = case action_configuration['type']
@@ -50,6 +50,20 @@ module Datadog
           configured_response || default_response(env)
         end
 
+        def graphql_response(gateway_multiplex)
+          multiplex_return = []
+          gateway_multiplex.queries.each do |query|
+            # This method is only called in places where GraphQL-Ruby is already required
+            query_result = ::GraphQL::Query::Result.new(
+              query: query,
+              values: JSON.parse(content('application/json'))
+            )
+            multiplex_return << query_result
+          end
+
+          multiplex_return
+        end
+
         private
 
         def default_response(env)

data/lib/datadog/appsec.rb

@@ -56,5 +56,6 @@ require_relative 'appsec/contrib/rack/integration'
 require_relative 'appsec/contrib/sinatra/integration'
 require_relative 'appsec/contrib/rails/integration'
 require_relative 'appsec/contrib/devise/integration'
+require_relative 'appsec/contrib/graphql/integration'
 
 require_relative 'appsec/autoload'

data/lib/datadog/core/configuration/components.rb

@@ -13,6 +13,7 @@ require_relative '../remote/component'
 require_relative '../../tracing/component'
 require_relative '../../profiling/component'
 require_relative '../../appsec/component'
+require_relative '../crashtracking/component'
 
 module Datadog
   module Core
@@ -56,19 +57,18 @@ module Datadog
           end
 
           def build_telemetry(settings, agent_settings, logger)
-            enabled = settings.telemetry.enabled
-            if agent_settings.adapter != Datadog::Core::Configuration::Ext::Agent::HTTP::ADAPTER
-              enabled = false
-              logger.debug { "Telemetry disabled. Agent network adapter not supported: #{agent_settings.adapter}" }
+            Telemetry::Component.build(settings, agent_settings, logger)
+          end
+
+          def build_crashtracker(settings, agent_settings, logger:)
+            return unless settings.crashtracking.enabled
+
+            if (libdatadog_api_failure = Datadog::Core::Crashtracking::Component::LIBDATADOG_API_FAILURE)
+              logger.debug("Cannot enable crashtracking: #{libdatadog_api_failure}")
+              return
             end
 
-            Telemetry::Component.new(
-              enabled: enabled,
-              metrics_enabled: enabled && settings.telemetry.metrics_enabled,
-              heartbeat_interval_seconds: settings.telemetry.heartbeat_interval_seconds,
-              metrics_aggregation_interval_seconds: settings.telemetry.metrics_aggregation_interval_seconds,
-              dependency_collection: settings.telemetry.dependency_collection
-            )
+            Datadog::Core::Crashtracking::Component.build(settings, agent_settings, logger: logger)
           end
         end
 
@@ -82,6 +82,7 @@ module Datadog
           :runtime_metrics,
           :telemetry,
           :tracer,
+          :crashtracker,
           :appsec
 
         def initialize(settings)
@@ -95,11 +96,12 @@ module Datadog
 
           @remote = Remote::Component.build(settings, agent_settings)
           @tracer = self.class.build_tracer(settings, agent_settings, logger: @logger)
+          @crashtracker = self.class.build_crashtracker(settings, agent_settings, logger: @logger)
 
           @profiler, profiler_logger_extra = Datadog::Profiling::Component.build_profiler_component(
             settings: settings,
             agent_settings: agent_settings,
-            optional_tracer: @tracer,
+            optional_tracer: @tracer
           )
           @environment_logger_extra.merge!(profiler_logger_extra) if profiler_logger_extra
 

data/lib/datadog/core/configuration/settings.rb

@@ -301,6 +301,16 @@ module Datadog
               o.default true
             end
 
+            # Can be used to enable/disable the Datadog::Profiling.allocation_count feature.
+            #
+            # Requires allocation profiling to be enabled.
+            #
+            # @default false
+            option :allocation_counting_enabled do |o|
+              o.type :bool
+              o.default false
+            end
+
             # Can be used to enable/disable the collection of heap profiles.
             #
             # This feature is alpha and disabled by default
@@ -441,14 +451,16 @@ module Datadog
               o.default 60
             end
 
-            # Enables reporting of information when the Ruby VM crashes.
-            #
-            # @default `DD_PROFILING_EXPERIMENTAL_CRASH_TRACKING_ENABLED` environment variable as a boolean,
-            # otherwise `false`
+            # DEV-3.0: Remove `experimental_crash_tracking_enabled` option
             option :experimental_crash_tracking_enabled do |o|
-              o.type :bool
-              o.env 'DD_PROFILING_EXPERIMENTAL_CRASH_TRACKING_ENABLED'
-              o.default false
+              o.after_set do |_, _, precedence|
+                unless precedence == Datadog::Core::Configuration::Option::Precedence::DEFAULT
+                  Core.log_deprecation(key: :experimental_crash_tracking_enabled) do
+                    'The profiling.advanced.experimental_crash_tracking_enabled setting has been deprecated for removal '\
+                    'and no longer does anything. Please remove it from your Datadog.configure block.'
+                  end
+                end
+              end
             end
           end
 
@@ -663,6 +675,24 @@ module Datadog
             o.type :bool
           end
 
+          # Enable agentless mode for telemetry: submit telemetry events directly to the intake without Datadog Agent.
+          #
+          # @return [Boolean]
+          # @!visibility private
+          option :agentless_enabled do |o|
+            o.type :bool
+            o.default false
+          end
+
+          # Overrides agentless telemetry URL. To be used internally for testing purposes only.
+          #
+          # @return [String]
+          # @!visibility private
+          option :agentless_url_override do |o|
+            o.type :string, nilable: true
+            o.env Core::Telemetry::Ext::ENV_AGENTLESS_URL_OVERRIDE
+          end
+
           # Enable metrics collection for telemetry. Metrics collection only works when telemetry is enabled and
           # metrics are enabled.
           # @default `DD_TELEMETRY_METRICS_ENABLED` environment variable, otherwise `true`.
@@ -734,6 +764,14 @@ module Datadog
             o.type :string, nilable: true
             o.env Core::Telemetry::Ext::ENV_INSTALL_TIME
           end
+
+          # Telemetry shutdown timeout in seconds
+          #
+          # @!visibility private
+          option :shutdown_timeout_seconds do |o|
+            o.type :float
+            o.default 1.0
+          end
         end
 
         # Remote configuration
@@ -794,6 +832,15 @@ module Datadog
           option :service
         end
 
+        settings :crashtracking do
+          # Enables reporting of information when Ruby VM crashes.
+          option :enabled do |o|
+            o.type :bool
+            o.default true
+            o.env 'DD_CRASHTRACKING_ENABLED'
+          end
+        end
+
         # TODO: Tracing should manage its own settings.
         #       Keep this extension here for now to keep things working.
         extend Datadog::Tracing::Configuration::Settings