datadog 2.2.0 → 2.4.0

data/lib/datadog/appsec/component.rb

@@ -10,10 +10,11 @@ module Datadog
     # Core-pluggable component for AppSec
     class Component
       class << self
-        def build_appsec_component(settings)
-          return unless settings.respond_to?(:appsec) && settings.appsec.enabled
+        def build_appsec_component(settings, telemetry:)
+          return if !settings.respond_to?(:appsec) || !settings.appsec.enabled
+          return if incompatible_ffi_version?
 
-          processor = create_processor(settings)
+          processor = create_processor(settings, telemetry)
 
           # We want to always instrument user events when AppSec is enabled.
           # There could be cases in which users use the DD_APPSEC_ENABLED Env variable to
@@ -28,8 +29,27 @@ module Datadog
 
         private
 
-        def create_processor(settings)
-          rules = AppSec::Processor::RuleLoader.load_rules(ruleset: settings.appsec.ruleset)
+        def incompatible_ffi_version?
+          ffi_version = Gem.loaded_specs['ffi'] && Gem.loaded_specs['ffi'].version
+          return true unless ffi_version
+
+          return false unless Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('3.3') &&
+            ffi_version < Gem::Version.new('1.16.0')
+
+          Datadog.logger.warn(
+            'AppSec is not supported in Ruby versions above 3.3.0 when using `ffi` versions older than 1.16.0, ' \
+            'and will be forcibly disabled due to a memory leak in `ffi`. ' \
+            'Please upgrade your `ffi` version to 1.16.0 or higher.'
+          )
+
+          true
+        end
+
+        def create_processor(settings, telemetry)
+          rules = AppSec::Processor::RuleLoader.load_rules(
+            telemetry: telemetry,
+            ruleset: settings.appsec.ruleset
+          )
           return nil unless rules
 
           actions = rules['actions']
@@ -47,9 +67,10 @@ module Datadog
             rules: [rules],
             data: data,
             exclusions: exclusions,
+            telemetry: telemetry
           )
 
-          processor = Processor.new(ruleset: ruleset)
+          processor = Processor.new(ruleset: ruleset, telemetry: telemetry)
           return nil unless processor.ready?
 
           processor
@@ -63,11 +84,11 @@ module Datadog
         @mutex = Mutex.new
       end
 
-      def reconfigure(ruleset:, actions:)
+      def reconfigure(ruleset:, actions:, telemetry:)
         @mutex.synchronize do
           AppSec::Processor::Actions.merge(actions)
 
-          new = Processor.new(ruleset: ruleset)
+          new = Processor.new(ruleset: ruleset, telemetry: telemetry)
 
           if new && new.ready?
             old = @processor

data/lib/datadog/appsec/configuration/settings.rb

@@ -9,8 +9,8 @@ module Datadog
       # Settings
       module Settings
         # rubocop:disable Layout/LineLength
-        DEFAULT_OBFUSCATOR_KEY_REGEX = '(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?)key)|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)|bearer|authorization'
-        DEFAULT_OBFUSCATOR_VALUE_REGEX = '(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?|access_?|secret_?)key(?:_?id)?|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)(?:\s*=[^;]|"\s*:\s*"[^"]+")|bearer\s+[a-z0-9\._\-]+|token:[a-z0-9]{13}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L][\w=-]+\.ey[I-L][\w=-]+(?:\.[\w.+\/=-]+)?|[\-]{5}BEGIN[a-z\s]+PRIVATE\sKEY[\-]{5}[^\-]+[\-]{5}END[a-z\s]+PRIVATE\sKEY|ssh-rsa\s*[a-z0-9\/\.+]{100,}'
+        DEFAULT_OBFUSCATOR_KEY_REGEX = '(?i)pass|pw(?:or)?d|secret|(?:api|private|public|access)[_-]?key|token|consumer[_-]?(?:id|key|secret)|sign(?:ed|ature)|bearer|authorization|jsessionid|phpsessid|asp\.net[_-]sessionid|sid|jwt'
+        DEFAULT_OBFUSCATOR_VALUE_REGEX = '(?i)(?:p(?:ass)?w(?:or)?d|pass(?:[_-]?phrase)?|secret(?:[_-]?key)?|(?:(?:api|private|public|access)[_-]?)key(?:[_-]?id)?|(?:(?:auth|access|id|refresh)[_-]?)?token|consumer[_-]?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?|jsessionid|phpsessid|asp\.net(?:[_-]|-)sessionid|sid|jwt)(?:\s*=[^;]|"\s*:\s*"[^"]+")|bearer\s+[a-z0-9\._\-]+|token:[a-z0-9]{13}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L][\w=-]+\.ey[I-L][\w=-]+(?:\.[\w.+\/=-]+)?|[\-]{5}BEGIN[a-z\s]+PRIVATE\sKEY[\-]{5}[^\-]+[\-]{5}END[a-z\s]+PRIVATE\sKEY|ssh-rsa\s*[a-z0-9\/\.+]{100,}'
         # rubocop:enable Layout/LineLength
         APPSEC_VALID_TRACK_USER_EVENTS_MODE = [
           'safe',

data/lib/datadog/appsec/contrib/devise/patcher/authenticatable_patch.rb

@@ -15,6 +15,7 @@ module Datadog
             def validate(resource, &block)
               result = super
               return result unless AppSec.enabled?
+              return result if @_datadog_skip_track_login_event
 
               track_user_events_configuration = Datadog.configuration.appsec.track_user_events
 

data/lib/datadog/appsec/contrib/devise/patcher/rememberable_patch.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Contrib
+      module Devise
+        module Patcher
+          # To avoid tracking new sessions that are created by
+          # Rememberable strategy as Login Success events.
+          module RememberablePatch
+            def validate(*args)
+              @_datadog_skip_track_login_event = true
+
+              super
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/devise/patcher.rb

@@ -2,6 +2,7 @@
 
 require_relative '../patcher'
 require_relative 'patcher/authenticatable_patch'
+require_relative 'patcher/rememberable_patch'
 require_relative 'patcher/registration_controller_patch'
 
 module Datadog
@@ -23,16 +24,25 @@ module Datadog
           end
 
           def patch
-            patch_authenticable_strategy
+            patch_authenticatable_strategy
+            patch_rememberable_strategy
             patch_registration_controller
 
             Patcher.instance_variable_set(:@patched, true)
           end
 
-          def patch_authenticable_strategy
+          def patch_authenticatable_strategy
             ::Devise::Strategies::Authenticatable.prepend(AuthenticatablePatch)
           end
 
+          def patch_rememberable_strategy
+            return unless ::Devise::STRATEGIES.include?(:rememberable)
+
+            # Rememberable strategy is required in autoloaded Rememberable model
+            ::Devise::Models::Rememberable # rubocop:disable Lint/Void
+            ::Devise::Strategies::Rememberable.prepend(RememberablePatch)
+          end
+
           def patch_registration_controller
             ::ActiveSupport.on_load(:after_initialize) do
               ::Devise::RegistrationsController.prepend(RegistrationControllerPatch)

data/lib/datadog/appsec/contrib/graphql/appsec_trace.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require 'json'
+require_relative 'gateway/multiplex'
+require_relative '../../instrumentation/gateway'
+
+module Datadog
+  module AppSec
+    module Contrib
+      module GraphQL
+        # These methods will be called by the GraphQL runtime to send the variables to the WAF.
+        # We actually don't need to create any span/trace.
+        module AppSecTrace
+          def execute_multiplex(multiplex:)
+            return super unless Datadog::AppSec.enabled?
+
+            gateway_multiplex = Gateway::Multiplex.new(multiplex)
+
+            multiplex_return, multiplex_response = Instrumentation.gateway.push('graphql.multiplex', gateway_multiplex) do
+              super
+            end
+
+            # Returns an error * the number of queries so that the entire multiplex is blocked
+            if multiplex_response
+              blocked_event = multiplex_response.find { |action, _options| action == :block }
+              multiplex_return = AppSec::Response.graphql_response(gateway_multiplex) if blocked_event
+            end
+
+            multiplex_return
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/graphql/gateway/multiplex.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require 'graphql'
+
+require_relative '../../../instrumentation/gateway/argument'
+
+module Datadog
+  module AppSec
+    module Contrib
+      module GraphQL
+        module Gateway
+          # Gateway Request argument. Normalized extration of data from Rack::Request
+          class Multiplex < Instrumentation::Gateway::Argument
+            attr_reader :multiplex
+
+            def initialize(multiplex)
+              super()
+              @multiplex = multiplex
+            end
+
+            def arguments
+              @arguments ||= build_arguments_hash
+            end
+
+            def queries
+              @multiplex.queries
+            end
+
+            private
+
+            # This method builds an array of argument hashes for each field with arguments in the query.
+            #
+            # For example, given the following query:
+            # query ($postSlug: ID = "my-first-post", $withComments: Boolean!) {
+            #   firstPost: post(slug: $postSlug) {
+            #     title
+            #     comments @include(if: $withComments) {
+            #       author { name }
+            #       content
+            #     }
+            #   }
+            # }
+            #
+            # The result would be:
+            # {"post"=>[{"slug"=>"my-first-post"}], "comments"=>[{"include"=>{"if"=>true}}]}
+            #
+            # Note that the `comments` "include" directive is included in the arguments list
+            def build_arguments_hash
+              queries.each_with_object({}) do |query, args_hash|
+                next unless query.selected_operation
+
+                arguments_from_selections(query.selected_operation.selections, query.variables, args_hash)
+              end
+            end
+
+            def arguments_from_selections(selections, query_variables, args_hash)
+              selections.each do |selection|
+                # rubocop:disable Style/ClassEqualityComparison
+                next unless selection.class.name == Integration::AST_NODE_CLASS_NAMES[:field]
+                # rubocop:enable Style/ClassEqualityComparison
+
+                selection_name = selection.alias || selection.name
+
+                if !selection.arguments.empty? || !selection.directives.empty?
+                  args_hash[selection_name] ||= []
+                  args_hash[selection_name] <<
+                    arguments_hash(selection.arguments, query_variables).merge!(
+                      arguments_from_directives(selection.directives, query_variables)
+                    )
+                end
+
+                arguments_from_selections(selection.selections, query_variables, args_hash)
+              end
+            end
+
+            def arguments_from_directives(directives, query_variables)
+              directives.each_with_object({}) do |directive, args_hash|
+                # rubocop:disable Style/ClassEqualityComparison
+                next unless directive.class.name == Integration::AST_NODE_CLASS_NAMES[:directive]
+                # rubocop:enable Style/ClassEqualityComparison
+
+                args_hash[directive.name] = arguments_hash(directive.arguments, query_variables)
+              end
+            end
+
+            def arguments_hash(arguments, query_variables)
+              arguments.each_with_object({}) do |argument, args_hash|
+                args_hash[argument.name] = argument_value(argument, query_variables)
+              end
+            end
+
+            def argument_value(argument, query_variables)
+              case argument.value.class.name
+              when Integration::AST_NODE_CLASS_NAMES[:variable_identifier]
+                # we need to pass query.variables here instead of query.provided_variables,
+                # since #provided_variables don't know anything about variable default value
+                query_variables[argument.value.name]
+              when Integration::AST_NODE_CLASS_NAMES[:input_object]
+                arguments_hash(argument.value.arguments, query_variables)
+              else
+                argument.value
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/graphql/gateway/watcher.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require 'json'
+require_relative '../../../instrumentation/gateway'
+require_relative '../reactive/multiplex'
+require_relative '../../../reactive/operation'
+
+module Datadog
+  module AppSec
+    module Contrib
+      module GraphQL
+        module Gateway
+          # Watcher for Rack gateway events
+          module Watcher
+            class << self
+              def watch
+                gateway = Instrumentation.gateway
+
+                watch_multiplex(gateway)
+              end
+
+              # This time we don't throw but use next
+              def watch_multiplex(gateway = Instrumentation.gateway)
+                gateway.watch('graphql.multiplex', :appsec) do |stack, gateway_multiplex|
+                  block = false
+                  event = nil
+
+                  scope = AppSec::Scope.active_scope
+
+                  if scope
+                    AppSec::Reactive::Operation.new('graphql.multiplex') do |op|
+                      GraphQL::Reactive::Multiplex.subscribe(op, scope.processor_context) do |result|
+                        event = {
+                          waf_result: result,
+                          trace: scope.trace,
+                          span: scope.service_entry_span,
+                          multiplex: gateway_multiplex,
+                          actions: result.actions
+                        }
+
+                        if scope.service_entry_span
+                          scope.service_entry_span.set_tag('appsec.blocked', 'true') if result.actions.include?('block')
+                          scope.service_entry_span.set_tag('appsec.event', 'true')
+                        end
+
+                        scope.processor_context.events << event
+                      end
+
+                      block = GraphQL::Reactive::Multiplex.publish(op, gateway_multiplex)
+                    end
+                  end
+
+                  next [nil, [[:block, event]]] if block
+
+                  ret, res = stack.call(gateway_multiplex.arguments)
+
+                  if event
+                    res ||= []
+                    res << [:monitor, event]
+                  end
+
+                  [ret, res]
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/graphql/integration.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require_relative '../integration'
+require_relative 'patcher'
+
+module Datadog
+  module AppSec
+    module Contrib
+      module GraphQL
+        # Description of GraphQL integration
+        class Integration
+          include Datadog::AppSec::Contrib::Integration
+
+          MINIMUM_VERSION = Gem::Version.new('2.0.19')
+
+          AST_NODE_CLASS_NAMES = {
+            field: 'GraphQL::Language::Nodes::Field',
+            directive: 'GraphQL::Language::Nodes::Directive',
+            variable_identifier: 'GraphQL::Language::Nodes::VariableIdentifier',
+            input_object: 'GraphQL::Language::Nodes::InputObject',
+          }.freeze
+
+          register_as :graphql, auto_patch: false
+
+          def self.version
+            Gem.loaded_specs['graphql'] && Gem.loaded_specs['graphql'].version
+          end
+
+          def self.loaded?
+            !defined?(::GraphQL).nil?
+          end
+
+          def self.compatible?
+            super && version >= MINIMUM_VERSION && ast_node_classes_defined?
+          end
+
+          def self.auto_instrument?
+            true
+          end
+
+          def self.ast_node_classes_defined?
+            AST_NODE_CLASS_NAMES.all? do |_, class_name|
+              Object.const_defined?(class_name)
+            end
+          end
+
+          def patcher
+            Patcher
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/graphql/patcher.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require_relative '../patcher'
+require_relative 'gateway/watcher'
+
+if Gem.loaded_specs['graphql'] && Gem.loaded_specs['graphql'].version >= Gem::Version.new('2.0.19')
+  require_relative 'appsec_trace'
+end
+
+module Datadog
+  module AppSec
+    module Contrib
+      module GraphQL
+        # Patcher for AppSec on GraphQL
+        module Patcher
+          include Datadog::AppSec::Contrib::Patcher
+
+          module_function
+
+          def patched?
+            Patcher.instance_variable_get(:@patched)
+          end
+
+          def target_version
+            Integration.version
+          end
+
+          def patch
+            Gateway::Watcher.watch
+            ::GraphQL::Schema.trace_with(AppSecTrace)
+            Patcher.instance_variable_set(:@patched, true)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/graphql/reactive/multiplex.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Contrib
+      module GraphQL
+        module Reactive
+          # Dispatch data from a GraphQL resolve query to the WAF context
+          module Multiplex
+            ADDRESSES = [
+              'graphql.server.all_resolvers'
+            ].freeze
+            private_constant :ADDRESSES
+
+            def self.publish(op, gateway_multiplex)
+              catch(:block) do
+                op.publish('graphql.server.all_resolvers', gateway_multiplex.arguments)
+
+                nil
+              end
+            end
+
+            def self.subscribe(op, waf_context)
+              op.subscribe(*ADDRESSES) do |*values|
+                Datadog.logger.debug { "reacted to #{ADDRESSES.inspect}: #{values.inspect}" }
+                arguments = values[0]
+
+                waf_args = {
+                  'graphql.server.all_resolvers' => arguments
+                }
+
+                waf_timeout = Datadog.configuration.appsec.waf_timeout
+                result = waf_context.run(waf_args, waf_timeout)
+
+                Datadog.logger.debug { "WAF TIMEOUT: #{result.inspect}" } if result.timeout
+
+                case result.status
+                when :match
+                  Datadog.logger.debug { "WAF: #{result.inspect}" }
+
+                  yield result
+                  throw(:block, true) unless result.actions.empty?
+                when :ok
+                  Datadog.logger.debug { "WAF OK: #{result.inspect}" }
+                when :invalid_call
+                  Datadog.logger.debug { "WAF CALL ERROR: #{result.inspect}" }
+                when :invalid_rule, :invalid_flow, :no_rule
+                  Datadog.logger.debug { "WAF RULE ERROR: #{result.inspect}" }
+                else
+                  Datadog.logger.debug { "WAF UNKNOWN: #{result.status.inspect} #{result.inspect}" }
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/rack/gateway/request.rb

@@ -41,18 +41,15 @@ module Datadog
 
             def headers
               result = request.env.each_with_object({}) do |(k, v), h|
-                h[k.gsub(/^HTTP_/, '').downcase!.tr('_', '-')] = v if k =~ /^HTTP_/
+                h[k.delete_prefix('HTTP_').tap(&:downcase!).tap { |s| s.tr!('_', '-') }] = v if k.start_with?('HTTP_')
               end
 
               result['content-type'] = request.content_type if request.content_type
-              result['content-length'] = request.content_length if request.content_length
+              # Since Rack 3.1, content-length is nil if the body is empty, but we still want to send it to the WAF.
+              result['content-length'] = request.content_length || '0'
               result
             end
 
-            def body
-              request.body.read.tap { request.body.rewind }
-            end
-
             def url
               request.url
             end

data/lib/datadog/appsec/event.rb

@@ -52,7 +52,7 @@ module Datadog
           # ensure rate limiter is called only when there are events to record
           return if events.empty? || span.nil?
 
-          Datadog::AppSec::RateLimiter.limit(:traces) do
+          Datadog::AppSec::RateLimiter.thread_local.limit do
             record_via_span(span, *events)
           end
         end

data/lib/datadog/appsec/processor/actions.rb

@@ -11,7 +11,7 @@ module Datadog
             @actions ||= []
           end
 
-          def fecth_configuration(action)
+          def fetch_configuration(action)
             actions.find { |action_configuration| action_configuration['id'] == action }
           end
 

data/lib/datadog/appsec/processor/rule_loader.rb

@@ -9,7 +9,7 @@ module Datadog
       # that load appsec rules and data from  settings
       module RuleLoader
         class << self
-          def load_rules(ruleset:)
+          def load_rules(ruleset:, telemetry:)
             begin
               case ruleset
               when :recommended, :strict
@@ -35,6 +35,8 @@ module Datadog
                 "libddwaf ruleset failed to load, ruleset: #{ruleset.inspect} error: #{e.inspect}"
               end
 
+              telemetry.report(e, description: 'libddwaf ruleset failed to load')
+
               nil
             end
           end

data/lib/datadog/appsec/processor/rule_merger.rb

@@ -18,25 +18,35 @@ module Datadog
           end
         end
 
-        DEFAULT_WAF_PROCESSORS = begin
-          JSON.parse(Datadog::AppSec::Assets.waf_processors)
-        rescue StandardError => e
-          Datadog.logger.error { "libddwaf rulemerger failed to parse default waf processors. Error: #{e.inspect}" }
-          []
-        end
-
-        DEFAULT_WAF_SCANNERS = begin
-          JSON.parse(Datadog::AppSec::Assets.waf_scanners)
-        rescue StandardError => e
-          Datadog.logger.error { "libddwaf rulemerger failed to parse default waf scanners. Error: #{e.inspect}" }
-          []
-        end
-
         class << self
+          # TODO: `processors` and `scanners` are not provided by the caller, consider removing them
           def merge(
+            telemetry:,
             rules:, data: [], overrides: [], exclusions: [], custom_rules: [],
-            processors: DEFAULT_WAF_PROCESSORS, scanners: DEFAULT_WAF_SCANNERS
+            processors: nil, scanners: nil
           )
+            processors ||= begin
+              default_waf_processors
+            rescue StandardError => e
+              Datadog.logger.error("libddwaf rulemerger failed to parse default waf processors. Error: #{e.inspect}")
+              telemetry.report(
+                e,
+                description: 'libddwaf rulemerger failed to parse default waf processors'
+              )
+              []
+            end
+
+            scanners ||= begin
+              default_waf_scanners
+            rescue StandardError => e
+              Datadog.logger.error("libddwaf rulemerger failed to parse default waf scanners. Error: #{e.inspect}")
+              telemetry.report(
+                e,
+                description: 'libddwaf rulemerger failed to parse default waf scanners'
+              )
+              []
+            end
+
             combined_rules = combine_rules(rules)
 
             combined_data = combine_data(data) if data.any?
@@ -53,6 +63,14 @@ module Datadog
             combined_rules
           end
 
+          def default_waf_processors
+            @default_waf_processors ||= JSON.parse(Datadog::AppSec::Assets.waf_processors)
+          end
+
+          def default_waf_scanners
+            @default_waf_scanners ||= JSON.parse(Datadog::AppSec::Assets.waf_scanners)
+          end
+
           private
 
           def combine_rules(rules)