datadog 2.0.0.beta1

data/lib/datadog/appsec/processor/rule_merger.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+require_relative '../assets'
+
+module Datadog
+  module AppSec
+    class Processor
+      # RuleMerger merge different sources of information
+      # into the rules payload
+      module RuleMerger
+        # RuleVersionMismatchError
+        class RuleVersionMismatchError < StandardError
+          def initialize(version1, version2)
+            msg = 'Merging rule files with different version could lead to unkown behaviour. '\
+              "We have receieve two rule files with versions: #{version1}, #{version2}. "\
+              'Please validate the configuration is correct and try again.'
+            super(msg)
+          end
+        end
+
+        DEFAULT_WAF_PROCESSORS = begin
+          JSON.parse(Datadog::AppSec::Assets.waf_processors)
+        rescue StandardError => e
+          Datadog.logger.error { "libddwaf rulemerger failed to parse default waf processors. Error: #{e.inspect}" }
+          []
+        end
+
+        DEFAULT_WAF_SCANNERS = begin
+          JSON.parse(Datadog::AppSec::Assets.waf_scanners)
+        rescue StandardError => e
+          Datadog.logger.error { "libddwaf rulemerger failed to parse default waf scanners. Error: #{e.inspect}" }
+          []
+        end
+
+        class << self
+          def merge(
+            rules:, data: [], overrides: [], exclusions: [], custom_rules: [],
+            processors: DEFAULT_WAF_PROCESSORS, scanners: DEFAULT_WAF_SCANNERS
+          )
+            combined_rules = combine_rules(rules)
+
+            combined_data = combine_data(data) if data.any?
+            combined_overrides = combine_overrides(overrides) if overrides.any?
+            combined_exclusions = combine_exclusions(exclusions) if exclusions.any?
+            combined_custom_rules = combine_custom_rules(custom_rules) if custom_rules.any?
+
+            combined_rules['rules_data'] = combined_data if combined_data
+            combined_rules['rules_override'] = combined_overrides if combined_overrides
+            combined_rules['exclusions'] = combined_exclusions if combined_exclusions
+            combined_rules['custom_rules'] = combined_custom_rules if combined_custom_rules
+            combined_rules['processors'] = processors
+            combined_rules['scanners'] = scanners
+            combined_rules
+          end
+
+          private
+
+          def combine_rules(rules)
+            return rules[0].dup if rules.size == 1
+
+            final_rules = []
+            # @type var final_version: ::String
+            final_version = (_ = nil)
+
+            rules.each do |rule_file|
+              version = rule_file['version']
+
+              if version && !final_version
+                final_version = version
+              elsif final_version != version
+                raise RuleVersionMismatchError.new(final_version, version)
+              end
+
+              final_rules.concat(rule_file['rules'])
+            end
+
+            {
+              'version' => final_version,
+              'rules' => final_rules
+            }
+          end
+
+          def combine_data(data)
+            result = []
+
+            data.each do |data_entry|
+              data_entry.each do |value|
+                existing_data = result.find { |x| x['id'] == value['id'] }
+
+                if existing_data && existing_data['type'] == value['type']
+                  # Duplicate entry base on type and id
+                  # We need to merge the existing data with the new one
+                  # and make sure to remove duplicates
+                  merged_data = merge_data_base_on_expiration(existing_data['data'], value['data'])
+                  existing_data['data'] = merged_data
+                else
+                  result << value
+                end
+              end
+            end
+
+            return unless result.any?
+
+            result
+          end
+
+          def merge_data_base_on_expiration(data1, data2)
+            result = data1.each_with_object({}) do |value, acc|
+              acc[value['value']] = value['expiration']
+            end
+
+            data2.each do |data|
+              if result.key?(data['value'])
+                # The value is duplicated so we need to keep
+                # the one with the highest expiration value
+                # We replace it if the expiration is higher than the current one
+                # or if no experiration
+                current_expiration = result[data['value']]
+                new_expiration = data['expiration']
+
+                if new_expiration.nil? || current_expiration && new_expiration > current_expiration
+                  result[data['value']] = new_expiration
+                end
+              else
+                result[data['value']] = data['expiration']
+              end
+            end
+
+            result.each_with_object([]) do |entry, acc|
+              value = { 'value' => entry[0] }
+              value['expiration'] = entry[1] if entry[1]
+
+              acc << value
+            end
+          end
+
+          def combine_overrides(overrides)
+            overrides.flatten
+          end
+
+          def combine_exclusions(exclusions)
+            exclusions.flatten
+          end
+
+          def combine_custom_rules(custom_rules)
+            custom_rules.flatten
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/processor.rb

@@ -0,0 +1,171 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    # Processor integrates libddwaf into datadog/appsec
+    class Processor
+      # Context manages a sequence of runs
+      class Context
+        attr_reader :time_ns, :time_ext_ns, :timeouts, :events
+
+        def initialize(processor)
+          @context = Datadog::AppSec::WAF::Context.new(processor.send(:handle))
+          @time_ns = 0.0
+          @time_ext_ns = 0.0
+          @timeouts = 0
+          @events = []
+          @run_mutex = Mutex.new
+        end
+
+        def run(input, timeout = WAF::LibDDWAF::DDWAF_RUN_TIMEOUT)
+          @run_mutex.lock
+
+          start_ns = Core::Utils::Time.get_time(:nanosecond)
+
+          input.reject! do |_, v|
+            case v
+            when TrueClass, FalseClass
+              false
+            else
+              v.nil? ? true : v.empty?
+            end
+          end
+
+          _code, res = @context.run(input, timeout)
+
+          stop_ns = Core::Utils::Time.get_time(:nanosecond)
+
+          # these updates are not thread safe and should be protected
+          @time_ns += res.total_runtime
+          @time_ext_ns += (stop_ns - start_ns)
+          @timeouts += 1 if res.timeout
+
+          res
+        ensure
+          @run_mutex.unlock
+        end
+
+        def extract_schema
+          return unless extract_schema?
+
+          input = {
+            'waf.context.processor' => {
+              'extract-schema' => true
+            }
+          }
+
+          _code, res = @context.run(input, WAF::LibDDWAF::DDWAF_RUN_TIMEOUT)
+
+          res
+        end
+
+        def finalize
+          @context.finalize
+        end
+
+        private
+
+        def extract_schema?
+          Datadog.configuration.appsec.api_security.enabled &&
+            Datadog.configuration.appsec.api_security.sample_rate.sample?
+        end
+      end
+
+      attr_reader :diagnostics, :addresses
+
+      def initialize(ruleset:)
+        @diagnostics = nil
+        @addresses = []
+        settings = Datadog.configuration.appsec
+
+        unless load_libddwaf && create_waf_handle(settings, ruleset)
+          Datadog.logger.warn { 'AppSec is disabled, see logged errors above' }
+        end
+      end
+
+      def ready?
+        !@handle.nil?
+      end
+
+      def finalize
+        @handle.finalize
+      end
+
+      protected
+
+      attr_reader :handle
+
+      private
+
+      def load_libddwaf
+        Processor.require_libddwaf && Processor.libddwaf_provides_waf?
+      end
+
+      def create_waf_handle(settings, ruleset)
+        # TODO: this may need to be reset if the main Datadog logging level changes after initialization
+        Datadog::AppSec::WAF.logger = Datadog.logger if Datadog.logger.debug? && settings.waf_debug
+
+        obfuscator_config = {
+          key_regex: settings.obfuscator_key_regex,
+          value_regex: settings.obfuscator_value_regex,
+        }
+
+        @handle = Datadog::AppSec::WAF::Handle.new(ruleset, obfuscator: obfuscator_config)
+        @diagnostics = @handle.diagnostics
+        @addresses = @handle.required_addresses
+
+        true
+      rescue WAF::LibDDWAF::Error => e
+        Datadog.logger.error do
+          "libddwaf failed to initialize, error: #{e.inspect}"
+        end
+
+        @diagnostics = e.diagnostics if e.diagnostics
+
+        false
+      rescue StandardError => e
+        Datadog.logger.error do
+          "libddwaf failed to initialize, error: #{e.inspect}"
+        end
+
+        false
+      end
+
+      class << self
+        # check whether libddwaf is required *and* able to provide the needed feature
+        def libddwaf_provides_waf?
+          defined?(Datadog::AppSec::WAF) ? true : false
+        end
+
+        # libddwaf raises a LoadError on unsupported platforms; it may at some
+        # point succeed in being required yet not provide a specific needed feature.
+        def require_libddwaf
+          Datadog.logger.debug { "libddwaf platform: #{libddwaf_platform}" }
+
+          require 'libddwaf'
+
+          true
+        rescue LoadError => e
+          Datadog.logger.error do
+            'libddwaf failed to load,' \
+              "installed platform: #{libddwaf_platform} ruby platforms: #{ruby_platforms} error: #{e.inspect}"
+          end
+
+          false
+        end
+
+        def libddwaf_spec
+          Gem.loaded_specs['libddwaf']
+        end
+
+        def libddwaf_platform
+          libddwaf_spec ? libddwaf_spec.platform.to_s : 'unknown'
+        end
+
+        def ruby_platforms
+          Gem.platforms.map(&:to_s)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/rate_limiter.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    # Simple per-thread rate limiter
+    # Since AppSec marks sampling to keep on a security event, this limits the flood of egress traces involving AppSec
+    class RateLimiter
+      def initialize(rate)
+        @rate = rate
+        @timestamps = []
+      end
+
+      def limit
+        now = Time.now.to_f
+
+        loop do
+          oldest = @timestamps.first
+
+          break if oldest.nil? || now - oldest < 1
+
+          @timestamps.shift
+        end
+
+        @timestamps << now
+
+        if (count = @timestamps.count) <= @rate
+          yield
+        else
+          Datadog.logger.debug { "Rate limit hit: #{count}/#{@rate} AppSec traces/second" }
+        end
+      end
+
+      class << self
+        def limit(name, &block)
+          rate_limiter(name).limit(&block)
+        end
+
+        # reset a rate limiter: used for testing
+        def reset!(name)
+          Thread.current[:datadog_security_trace_rate_limiter] = nil
+        end
+
+        protected
+
+        def rate_limiter(name)
+          case name
+          when :traces
+            Thread.current[:datadog_security_trace_rate_limiter] ||= RateLimiter.new(trace_rate_limit)
+          else
+            raise "unsupported rate limiter: #{name.inspect}"
+          end
+        end
+
+        def trace_rate_limit
+          Datadog.configuration.appsec.trace_rate_limit
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/reactive/address_hash.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Reactive
+      # AddressHash for Reactive Engine
+      class AddressHash < Hash
+        def self.new(*arguments, &block)
+          super { |h, k| h[k] = [] }
+        end
+
+        def addresses
+          keys.flatten
+        end
+
+        def with(address)
+          keys.select { |k| k.include?(address) }
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/reactive/engine.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require_relative 'address_hash'
+require_relative 'subscriber'
+
+module Datadog
+  module AppSec
+    module Reactive
+      # Reactive Engine
+      class Engine
+        def initialize
+          @data = {}
+          @subscribers = AddressHash.new
+        end
+
+        def subscribe(*addresses, &block)
+          @subscribers[addresses.freeze] << Subscriber.new(&block)
+        end
+
+        def publish(address, value)
+          # check if someone has address subscribed
+          if @subscribers.addresses.include?(address)
+
+            # someone will be interested, set value
+            @data[address] = value
+
+            # find candidates i.e address groups that contain the just posted address
+            @subscribers.with(address).each do |addresses|
+              # find targets to the address group containing the posted address
+              subscribers = @subscribers[addresses]
+
+              # is all data for the targets available?
+              if (addresses - @data.keys).empty?
+                hash = addresses.each_with_object({}) { |a, h| h[a] = @data[a] }
+                subscribers.each { |s| s.call(*hash.values) }
+              end
+            end
+          end
+        end
+
+        private
+
+        attr_reader :subscribers, :data
+      end
+    end
+  end
+end

data/lib/datadog/appsec/reactive/operation.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require_relative 'engine'
+
+module Datadog
+  module AppSec
+    module Reactive
+      # Reactive Engine nested operation tracking
+      class Operation
+        attr_reader :reactive,
+          :parent,
+          :name
+
+        def initialize(name, parent = nil, reactive_engine = nil)
+          Datadog.logger.debug { "operation: #{name} initialize" }
+          @name = name
+          @parent = parent
+          @reactive = select_reactive_engine(reactive_engine, parent)
+
+          # TODO: concurrent store
+          # TODO: constant
+          Thread.current[:datadog_security_active_operation] = self
+
+          yield self if block_given?
+        ensure
+          finalize
+        end
+
+        # TODO: use structs instead of an arg splat
+        def subscribe(*addresses, &block)
+          reactive.subscribe(*addresses, &block)
+        end
+
+        def publish(address, data)
+          reactive.publish(address, data)
+        end
+
+        def finalize
+          Datadog.logger.debug { "operation: #{name} finalize" }
+          Thread.current[:datadog_security_active_operation] = parent
+        end
+
+        private
+
+        def select_reactive_engine(reactive, parent)
+          return reactive if reactive
+
+          return parent.reactive unless parent.nil?
+
+          Reactive::Engine.new
+        end
+
+        class << self
+          def active
+            Thread.current[:datadog_security_active_operation]
+          end
+
+          private
+
+          # For testing only.
+          def reset!
+            Thread.current[:datadog_security_active_operation] = nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/reactive/subscriber.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Reactive
+      # Reactive Engine subscriber
+      class Subscriber
+        def initialize(&block)
+          @block = block
+          freeze
+        end
+
+        def call(*args)
+          @block.call(*args)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/remote.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+require_relative '../core/remote/dispatcher'
+require_relative 'processor/rule_merger'
+require_relative 'processor/rule_loader'
+
+module Datadog
+  module AppSec
+    # Remote
+    module Remote
+      class ReadError < StandardError; end
+      class NoRulesError < StandardError; end
+
+      class << self
+        CAP_ASM_RESERVED_1                = 1 << 0   # RESERVED
+        CAP_ASM_ACTIVATION                = 1 << 1   # Remote activation via ASM_FEATURES product
+        CAP_ASM_IP_BLOCKING               = 1 << 2   # accept IP blocking data from ASM_DATA product
+        CAP_ASM_DD_RULES                  = 1 << 3   # read ASM rules from ASM_DD product
+        CAP_ASM_EXCLUSIONS                = 1 << 4   # exclusion filters (passlist) via ASM product
+        CAP_ASM_REQUEST_BLOCKING          = 1 << 5   # can block on request info
+        CAP_ASM_RESPONSE_BLOCKING         = 1 << 6   # can block on response info
+        CAP_ASM_USER_BLOCKING             = 1 << 7   # accept user blocking data from ASM_DATA product
+        CAP_ASM_CUSTOM_RULES              = 1 << 8   # accept custom rules
+        CAP_ASM_CUSTOM_BLOCKING_RESPONSE  = 1 << 9   # supports custom http code or redirect sa blocking response
+        CAP_ASM_TRUSTED_IPS               = 1 << 10  # supports trusted ip
+
+        # TODO: we need to dynamically add CAP_ASM_ACTIVATION once we support it
+        ASM_CAPABILITIES = [
+          CAP_ASM_IP_BLOCKING,
+          CAP_ASM_USER_BLOCKING,
+          CAP_ASM_EXCLUSIONS,
+          CAP_ASM_REQUEST_BLOCKING,
+          CAP_ASM_RESPONSE_BLOCKING,
+          CAP_ASM_DD_RULES,
+          CAP_ASM_CUSTOM_RULES,
+          CAP_ASM_CUSTOM_BLOCKING_RESPONSE,
+          CAP_ASM_TRUSTED_IPS,
+        ].freeze
+
+        ASM_PRODUCTS = [
+          'ASM_DD',       # Datadog employee issued configuration
+          'ASM',          # customer issued configuration (rulesets, passlist...)
+          'ASM_FEATURES', # capabilities
+          'ASM_DATA',     # config files (IP addresses or users for blocking)
+        ].freeze
+
+        def capabilities
+          remote_features_enabled? ? ASM_CAPABILITIES : []
+        end
+
+        def products
+          remote_features_enabled? ? ASM_PRODUCTS : []
+        end
+
+        # rubocop:disable Metrics/MethodLength
+        def receivers
+          return [] unless remote_features_enabled?
+
+          matcher = Core::Remote::Dispatcher::Matcher::Product.new(ASM_PRODUCTS)
+          receiver = Core::Remote::Dispatcher::Receiver.new(matcher) do |repository, changes|
+            changes.each do |change|
+              Datadog.logger.debug { "remote config change: '#{change.path}'" }
+            end
+
+            rules = []
+            custom_rules = []
+            data = []
+            overrides = []
+            exclusions = []
+            actions = []
+
+            repository.contents.each do |content|
+              parsed_content = parse_content(content)
+
+              case content.path.product
+              when 'ASM_DD'
+                rules << parsed_content
+              when 'ASM_DATA'
+                data << parsed_content['rules_data'] if parsed_content['rules_data']
+              when 'ASM'
+                overrides << parsed_content['rules_override'] if parsed_content['rules_override']
+                exclusions << parsed_content['exclusions'] if parsed_content['exclusions']
+                custom_rules << parsed_content['custom_rules'] if parsed_content['custom_rules']
+                actions.concat(parsed_content['actions']) if parsed_content['actions']
+              end
+            end
+
+            if rules.empty?
+              settings_rules = AppSec::Processor::RuleLoader.load_rules(ruleset: Datadog.configuration.appsec.ruleset)
+
+              raise NoRulesError, 'no default rules available' unless settings_rules
+
+              rules = [settings_rules]
+            end
+
+            ruleset = AppSec::Processor::RuleMerger.merge(
+              rules: rules,
+              data: data,
+              overrides: overrides,
+              exclusions: exclusions,
+              custom_rules: custom_rules,
+            )
+
+            Datadog::AppSec.reconfigure(ruleset: ruleset, actions: actions)
+          end
+
+          [receiver]
+        end
+        # rubocop:enable Metrics/MethodLength
+
+        private
+
+        def remote_features_enabled?
+          Datadog.configuration.appsec.using_default?(:ruleset)
+        end
+
+        def parse_content(content)
+          data = content.data.read
+
+          content.data.rewind
+
+          raise ReadError, 'EOF reached' if data.nil?
+
+          JSON.parse(data)
+        end
+      end
+    end
+  end
+end