ddtrace 1.10.0 → 1.11.0.beta1

data/lib/datadog/appsec/configuration.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'configuration/settings'
 
 module Datadog
@@ -82,6 +84,12 @@ module Datadog
         def settings
           @settings ||= Settings.new
         end
+
+        private
+
+        def default_setting?(setting)
+          settings.send(:default?, setting)
+        end
       end
     end
   end

data/lib/datadog/appsec/contrib/auto_instrument.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Datadog
   module AppSec
     module Contrib

data/lib/datadog/appsec/contrib/configuration/settings.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative '../../../core/configuration/base'
 
 module Datadog

data/lib/datadog/appsec/contrib/integration.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Datadog
   module AppSec
     module Contrib

data/lib/datadog/appsec/contrib/patcher.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Datadog
   module AppSec
     module Contrib

data/lib/datadog/appsec/contrib/rack/configuration/settings.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative '../../configuration/settings'
 require_relative '../ext'
 

data/lib/datadog/appsec/contrib/rack/gateway/request.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
 require_relative '../../../instrumentation/gateway/argument'
+require_relative '../../../../core/header_collection'
 require_relative '../../../../tracing/client_ip'
-require_relative '../../../../tracing/contrib/rack/header_collection'
 
 module Datadog
   module AppSec
@@ -14,7 +14,7 @@ module Datadog
             attr_reader :env
 
             def initialize(env)
-              super
+              super()
               @env = env
             end
 
@@ -33,20 +33,9 @@ module Datadog
               end
             end
 
-            # Rack < 2.0 does not have :each_header
-            # TODO: We need access to Rack here. We must make sure we are able to load AppSec without Rack,
-            # TODO: while still ensure correctness in ths code path.
-            if defined?(::Rack) && ::Rack::Request.instance_methods.include?(:each_header)
-              def headers
-                request.each_header.each_with_object({}) do |(k, v), h|
-                  h[k.gsub(/^HTTP_/, '').downcase.tr('_', '-')] = v if k =~ /^HTTP_/
-                end
-              end
-            else
-              def headers
-                request.env.each_with_object({}) do |(k, v), h|
-                  h[k.gsub(/^HTTP_/, '').downcase.tr('_', '-')] = v if k =~ /^HTTP_/
-                end
+            def headers
+              request.env.each_with_object({}) do |(k, v), h|
+                h[k.gsub(/^HTTP_/, '').downcase.tr('_', '-')] = v if k =~ /^HTTP_/
               end
             end
 
@@ -85,16 +74,9 @@ module Datadog
 
             def client_ip
               remote_ip = remote_addr
-              headers = Datadog::Tracing::Contrib::Rack::Header::RequestHeaderCollection.new(env)
-
-              result = Datadog::Tracing::ClientIp.raw_ip_from_request(headers, remote_ip)
+              header_collection = Datadog::Core::HeaderCollection.from_hash(headers)
 
-              if result.raw_ip
-                ip = Datadog::Tracing::ClientIp.strip_decorations(result.raw_ip)
-                return unless Datadog::Tracing::ClientIp.valid_ip?(ip)
-
-                ip
-              end
+              Datadog::Tracing::ClientIp.extract_client_ip(header_collection, remote_ip)
             end
           end
         end

data/lib/datadog/appsec/contrib/rack/gateway/response.rb

@@ -12,7 +12,7 @@ module Datadog
             attr_reader :body, :status, :headers, :active_context
 
             def initialize(body, status, headers, active_context:)
-              super
+              super()
               @body = body
               @status = status
               @headers = headers.each_with_object({}) { |(k, v), h| h[k.downcase] = v }

data/lib/datadog/appsec/contrib/rack/patcher.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative '../patcher'
 require_relative '../../monitor'
 require_relative 'gateway/watcher'

data/lib/datadog/appsec/contrib/rack/reactive/response.rb

@@ -9,12 +9,14 @@ module Datadog
           module Response
             ADDRESSES = [
               'response.status',
+              'response.headers',
             ].freeze
             private_constant :ADDRESSES
 
             def self.publish(op, gateway_response)
               catch(:block) do
                 op.publish('response.status', gateway_response.status)
+                op.publish('response.headers', gateway_response.headers)
 
                 nil
               end
@@ -25,9 +27,13 @@ module Datadog
                 Datadog.logger.debug { "reacted to #{ADDRESSES.inspect}: #{values.inspect}" }
 
                 response_status = values[0]
+                response_headers = values[1]
+                response_headers_no_cookies = response_headers.dup.tap { |h| h.delete('set-cookie') }
 
                 waf_args = {
                   'server.response.status' => response_status.to_s,
+                  'server.response.headers' => response_headers,
+                  'server.response.headers.no_cookies' => response_headers_no_cookies,
                 }
 
                 waf_timeout = Datadog::AppSec.settings.waf_timeout

data/lib/datadog/appsec/contrib/rack/request_middleware.rb

@@ -23,17 +23,30 @@ module Datadog
             @oneshot_tags_sent = false
           end
 
+          # rubocop:disable Metrics/AbcSize,Metrics/PerceivedComplexity,Metrics/CyclomaticComplexity,Metrics/MethodLength
           def call(env)
             return @app.call(env) unless Datadog::AppSec.enabled?
 
+            Datadog::Core::Remote.active_remote.barrier(:once) unless Datadog::Core::Remote.active_remote.nil?
             processor = Datadog::AppSec.processor
 
-            return @app.call(env) if processor.nil? || !processor.ready?
+            processor = nil
+            ready = false
+            context = nil
+
+            Datadog::AppSec.reconfigure_lock do
+              processor = Datadog::AppSec.processor
+
+              if !processor.nil? && processor.ready?
+                context = processor.activate_context
+                env['datadog.waf.context'] = context
+                ready = true
+              end
+            end
 
             # TODO: handle exceptions, except for @app.call
 
-            context = processor.activate_context
-            env['datadog.waf.context'] = context
+            return @app.call(env) unless ready
 
             gateway_request = Gateway::Request.new(env)
 
@@ -76,6 +89,7 @@ module Datadog
               processor.deactivate_context
             end
           end
+          # rubocop:enable Metrics/AbcSize,Metrics/PerceivedComplexity,Metrics/CyclomaticComplexity,Metrics/MethodLength
 
           private
 

data/lib/datadog/appsec/contrib/rails/configuration/settings.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative '../../configuration/settings'
 require_relative '../ext'
 

data/lib/datadog/appsec/contrib/rails/framework.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Datadog
   module AppSec
     module Contrib

data/lib/datadog/appsec/contrib/rails/gateway/request.rb

@@ -12,7 +12,7 @@ module Datadog
             attr_reader :request
 
             def initialize(request)
-              super
+              super()
               @request = request
             end
 

data/lib/datadog/appsec/contrib/rails/request.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Datadog
   module AppSec
     module Contrib

data/lib/datadog/appsec/contrib/rails/request_middleware.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Datadog
   module AppSec
     module Contrib

data/lib/datadog/appsec/contrib/sinatra/configuration/settings.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative '../../configuration/settings'
 require_relative '../ext'
 

data/lib/datadog/appsec/contrib/sinatra/framework.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Datadog
   module AppSec
     module Contrib

data/lib/datadog/appsec/contrib/sinatra/gateway/route_params.rb

@@ -12,7 +12,7 @@ module Datadog
             attr_reader :params
 
             def initialize(params)
-              super
+              super()
               @params = params
             end
           end

data/lib/datadog/appsec/contrib/sinatra/request_middleware.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Datadog
   module AppSec
     module Contrib

data/lib/datadog/appsec/extensions.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'configuration'
 
 module Datadog

data/lib/datadog/appsec/instrumentation/gateway/argument.rb

@@ -5,16 +5,14 @@ module Datadog
     module Instrumentation
       class Gateway
         # Base class for Gateway Arguments
-        class Argument
-          def initialize(*); end
-        end
+        class Argument; end # rubocop:disable Lint/EmptyClass
 
         # Gateway User argument
         class User < Argument
           attr_reader :id
 
           def initialize(id)
-            super
+            super()
             @id = id
           end
         end

data/lib/datadog/appsec/processor/rule_loader.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require_relative '../assets'
+
+module Datadog
+  module AppSec
+    class Processor
+      # RuleLoader utility modules
+      # that load appsec rules and data from  settings
+      module RuleLoader
+        class << self
+          def load_rules(ruleset:)
+            begin
+              case ruleset
+              when :recommended, :strict
+                JSON.parse(Datadog::AppSec::Assets.waf_rules(ruleset))
+              when :risky
+                Datadog.logger.warn(
+                  'The :risky Application Security Management ruleset has been deprecated and no longer available.'\
+                  'The `:recommended` ruleset will be used instead.'\
+                  'Please remove the `appsec.ruleset = :risky` setting from your Datadog.configure block.'
+                )
+                JSON.parse(Datadog::AppSec::Assets.waf_rules(:recommended))
+              when String
+                JSON.parse(File.read(File.expand_path(ruleset)))
+              when File, StringIO
+                JSON.parse(ruleset.read || '').tap { ruleset.rewind }
+              when Hash
+                ruleset
+              else
+                raise ArgumentError, "unsupported value for ruleset setting: #{ruleset.inspect}"
+              end
+            rescue StandardError => e
+              Datadog.logger.error do
+                "libddwaf ruleset failed to load, ruleset: #{ruleset.inspect} error: #{e.inspect}"
+              end
+
+              nil
+            end
+          end
+
+          def load_data(ip_denylist: [], user_id_denylist: [])
+            data = []
+            data << { 'rules_data' => [denylist_data('blocked_ips', ip_denylist)] } if ip_denylist.any?
+            data << { 'rules_data' => [denylist_data('blocked_users', user_id_denylist)] } if user_id_denylist.any?
+
+            data.any? ? data : nil
+          end
+
+          private
+
+          def denylist_data(id, denylist)
+            {
+              'id' => id,
+              'type' => 'data_with_expiration',
+              'data' => denylist.map { |v| { 'value' => v.to_s, 'expiration' => 2**63 } }
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/processor/rule_merger.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    class Processor
+      # RuleMerger merge different sources of information
+      # into the rules payload
+      module RuleMerger
+        # RuleVersionMismatchError
+        class RuleVersionMismatchError < StandardError
+          def initialize(version1, version2)
+            msg = 'Merging rule files with different version could lead to unkown behaviour. '\
+              "We have receieve two rule files with versions: #{version1}, #{version2}. "\
+              'Please validate the configuration is correct and try again.'
+            super(msg)
+          end
+        end
+
+        class << self
+          def merge(rules:, data: nil, overrides: nil, exclusions: nil)
+            combined_rules = combine_rules(rules)
+
+            rules_data = combine_data(data) if data
+            rules_overrides = combine_overrides(overrides) if overrides
+            rules_exclusions = combine_exclusions(exclusions) if exclusions
+
+            combined_rules['rules_data'] = rules_data if rules_data
+            combined_rules['rules_override'] = rules_overrides if rules_overrides
+            combined_rules['exclusions'] = rules_exclusions if rules_exclusions
+
+            combined_rules
+          end
+
+          private
+
+          def combine_rules(rules)
+            return rules[0].dup if rules.size == 1
+
+            final_rules = []
+            # @type var final_version: ::String
+            final_version = (_ = nil)
+
+            rules.each do |rule_file|
+              version = rule_file['version']
+
+              if version && !final_version
+                final_version = version
+              elsif final_version != version
+                raise RuleVersionMismatchError.new(final_version, version)
+              end
+
+              final_rules.concat(rule_file['rules'])
+            end
+
+            {
+              'version' => final_version,
+              'rules' => final_rules
+            }
+          end
+
+          def combine_data(data)
+            result = []
+
+            data.each do |data_entry|
+              data_entry['rules_data'].each do |value|
+                existing_data = result.find { |x| x['id'] == value['id'] }
+
+                if existing_data && existing_data['type'] == value['type']
+                  # Duplicate entry base on type and id
+                  # We need to merge the existing data with the new one
+                  # and make sure to remove duplicates
+                  merged_data = merge_data_base_on_expiration(existing_data['data'], value['data'])
+                  existing_data['data'] = merged_data
+                else
+                  result << value
+                end
+              end
+            end
+
+            return unless result.any?
+
+            result
+          end
+
+          def merge_data_base_on_expiration(data1, data2)
+            result = data1.each_with_object({}) do |value, acc|
+              acc[value['value']] = value['expiration']
+            end
+
+            data2.each do |data|
+              if result.key?(data['value'])
+                # The value is duplicated so we need to keep
+                # the one with the highest expiration value
+                # We replace it if the expiration is higher than the current one
+                # or if no experiration
+                current_expiration = result[data['value']]
+                new_expiration = data['expiration']
+
+                if new_expiration.nil? || current_expiration && new_expiration > current_expiration
+                  result[data['value']] = new_expiration
+                end
+              else
+                result[data['value']] = data['expiration']
+              end
+            end
+
+            result.each_with_object([]) do |entry, acc|
+              value = { 'value' => entry[0] }
+              value['expiration'] = entry[1] if entry[1]
+
+              acc << value
+            end
+          end
+
+          def combine_overrides(overrides)
+            rules_override = []
+
+            overrides.each do |override|
+              override['rules_override'].each do |rule_override|
+                rules_override << rule_override
+              end
+            end
+
+            return if rules_override.empty?
+
+            rules_override
+          end
+
+          def combine_exclusions(exclusions)
+            rules_exclusions = []
+
+            exclusions.each do |exclusion|
+              exclusion['exclusions'].each do |rule_exclusion|
+                rules_exclusions << rule_exclusion
+              end
+            end
+
+            return if rules_exclusions.empty?
+
+            rules_exclusions
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/processor.rb

@@ -1,4 +1,4 @@
-require_relative 'assets'
+# frozen_string_literal: true
 
 module Datadog
   module AppSec
@@ -14,22 +14,28 @@ module Datadog
           @time_ext_ns = 0.0
           @timeouts = 0
           @events = []
+          @run_mutex = Mutex.new
         end
 
         def run(input, timeout = WAF::LibDDWAF::DDWAF_RUN_TIMEOUT)
+          @run_mutex.lock
+
           start_ns = Core::Utils::Time.get_time(:nanosecond)
 
+          # this WAF::Context#run call is not thread safe as it mutates the context
           # TODO: remove multiple assignment
-          _code, res = _ = @context.run(input, timeout)
-          # @type var res: WAF::Result
+          _code, res = @context.run(input, timeout)
 
           stop_ns = Core::Utils::Time.get_time(:nanosecond)
 
+          # these updates are not thread safe and should be protected
           @time_ns += res.total_runtime
           @time_ext_ns += (stop_ns - start_ns)
           @timeouts += 1 if res.timeout
 
           res
+        ensure
+          @run_mutex.unlock
         end
 
         def finalize
@@ -63,22 +69,18 @@ module Datadog
 
       attr_reader :ruleset_info, :addresses
 
-      def initialize
+      def initialize(ruleset:)
         @ruleset_info = nil
         @addresses = []
         settings = Datadog::AppSec.settings
 
-        unless load_libddwaf && load_ruleset(settings) && create_waf_handle(settings)
+        unless load_libddwaf && create_waf_handle(settings, ruleset)
           Datadog.logger.warn { 'AppSec is disabled, see logged errors above' }
-
-          return
         end
-
-        apply_denylist_data(settings)
       end
 
       def ready?
-        !@ruleset.nil? && !@handle.nil?
+        !@handle.nil?
       end
 
       def new_context
@@ -102,14 +104,6 @@ module Datadog
         context.finalize
       end
 
-      def update_rule_data(data)
-        @handle.update_rule_data(data)
-      end
-
-      def toggle_rules(map)
-        @handle.toggle_rules(map)
-      end
-
       def finalize
         @handle.finalize
       end
@@ -120,61 +114,11 @@ module Datadog
 
       private
 
-      def apply_denylist_data(settings)
-        ruledata_setting = []
-        ruledata_setting << denylist_data('blocked_ips', settings.ip_denylist)
-        ruledata_setting << denylist_data('blocked_users', settings.user_id_denylist)
-
-        update_rule_data(ruledata_setting)
-      end
-
-      def denylist_data(id, denylist)
-        {
-          'id' => id,
-          'type' => 'data_with_expiration',
-          'data' => denylist.map { |v| { 'value' => v.to_s, 'expiration' => 2**63 } }
-        }
-      end
-
       def load_libddwaf
         Processor.require_libddwaf && Processor.libddwaf_provides_waf?
       end
 
-      def load_ruleset(settings)
-        ruleset_setting = settings.ruleset
-
-        begin
-          @ruleset = case ruleset_setting
-                     when :recommended, :strict
-                       JSON.parse(Datadog::AppSec::Assets.waf_rules(ruleset_setting))
-                     when :risky
-                       JSON.parse(Datadog::AppSec::Assets.waf_rules(:recommended))
-                       Datadog.logger.warn(
-                         'The :risky Application Security Management ruleset has been deprecated and no longer available.'\
-                         'The `:recommended` ruleset will be used instead.'\
-                         'Please remove the `appsec.ruleset = :risky` setting from your Datadog.configure block.'
-                       )
-                     when String
-                       JSON.parse(File.read(ruleset_setting))
-                     when File, StringIO
-                       JSON.parse(ruleset_setting.read || '').tap { ruleset_setting.rewind }
-                     when Hash
-                       ruleset_setting
-                     else
-                       raise ArgumentError, "unsupported value for ruleset setting: #{ruleset_setting.inspect}"
-                     end
-
-          true
-        rescue StandardError => e
-          Datadog.logger.error do
-            "libddwaf ruleset failed to load, ruleset: #{ruleset_setting.inspect} error: #{e.inspect}"
-          end
-
-          false
-        end
-      end
-
-      def create_waf_handle(settings)
+      def create_waf_handle(settings, ruleset)
         # TODO: this may need to be reset if the main Datadog logging level changes after initialization
         Datadog::AppSec::WAF.logger = Datadog.logger if Datadog.logger.debug? && settings.waf_debug
 
@@ -182,7 +126,8 @@ module Datadog
           key_regex: settings.obfuscator_key_regex,
           value_regex: settings.obfuscator_value_regex,
         }
-        @handle = Datadog::AppSec::WAF::Handle.new(@ruleset, obfuscator: obfuscator_config)
+
+        @handle = Datadog::AppSec::WAF::Handle.new(ruleset, obfuscator: obfuscator_config)
         @ruleset_info = @handle.ruleset_info
         @addresses = @handle.required_addresses