cw-datadog 2.23.0.2

data/lib/datadog/appsec/event.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+require 'json'
+require_relative 'rate_limiter'
+require_relative 'trace_keeper'
+require_relative 'compressed_json'
+
+module Datadog
+  module AppSec
+    # AppSec event
+    module Event
+      ATTRIBUTES_SCHEMA_KEY_PREFIX = '_dd.appsec.s.'
+      ATTRIBUTES_SCHEMA_MAX_COMPRESSED_SIZE = 25000
+      ALLOWED_REQUEST_HEADERS = %w[
+        x-forwarded-for
+        x-client-ip
+        x-real-ip
+        x-forwarded
+        x-cluster-client-ip
+        forwarded-for
+        forwarded
+        via
+        true-client-ip
+        content-length
+        content-type
+        content-encoding
+        content-language
+        host
+        user-agent
+        accept
+        accept-encoding
+        accept-language
+      ].freeze
+
+      ALLOWED_RESPONSE_HEADERS = %w[
+        content-length
+        content-type
+        content-encoding
+        content-language
+      ].freeze
+
+      class << self
+        def tag(context, waf_result)
+          return if context.span.nil?
+
+          if waf_result.actions.key?('block_request') || waf_result.actions.key?('redirect_request')
+            context.span.set_tag('appsec.blocked', 'true')
+          end
+
+          context.span.set_tag('appsec.event', 'true')
+        end
+
+        def record(context, request: nil, response: nil)
+          return if context.events.empty? || context.span.nil?
+
+          Datadog::AppSec::RateLimiter.thread_local.limit do
+            context.events.group_by(&:trace).each do |trace, event_group|
+              unless trace
+                next Datadog.logger.debug do
+                  "AppSec: Cannot record event group with #{event_group.count} events because it has no trace"
+                end
+              end
+
+              if event_group.any? { |event| event.keep? || event.schema? }
+                TraceKeeper.keep!(trace)
+
+                context.span['_dd.origin'] = 'appsec'
+                context.span.set_tags(request_tags(request)) if request
+                context.span.set_tags(response_tags(response)) if response
+              end
+
+              context.span.set_tags(waf_tags(event_group))
+            end
+          end
+        end
+
+        private
+
+        def request_tags(request)
+          tags = {}
+
+          tags['http.host'] = request.host if request.host
+          tags['http.useragent'] = request.user_agent if request.user_agent
+          tags['network.client.ip'] = request.remote_addr if request.remote_addr
+
+          request.headers.each_with_object(tags) do |(name, value), memo|
+            next unless ALLOWED_REQUEST_HEADERS.include?(name)
+
+            memo["http.request.headers.#{name}"] = value
+          end
+        end
+
+        def response_tags(response)
+          response.headers.each_with_object({}) do |(name, value), memo|
+            next unless ALLOWED_RESPONSE_HEADERS.include?(name)
+
+            memo["http.response.headers.#{name}"] = value
+          end
+        end
+
+        def waf_tags(security_events)
+          triggers = []
+
+          tags = security_events.each_with_object({}) do |security_event, memo|
+            triggers.concat(security_event.waf_result.events)
+
+            security_event.waf_result.attributes.each do |key, value|
+              next memo[key] = value unless key.start_with?(ATTRIBUTES_SCHEMA_KEY_PREFIX)
+
+              value = CompressedJson.dump(value)
+              next if value.nil?
+
+              if value.size >= ATTRIBUTES_SCHEMA_MAX_COMPRESSED_SIZE
+                Datadog.logger.debug { "AppSec: Schema key '#{key}' will not be included into span tags due to it's size" }
+                next
+              end
+
+              memo[key] = value
+            end
+          end
+
+          tags['_dd.appsec.json'] = json_parse({triggers: triggers}) unless triggers.empty?
+          tags
+        end
+
+        # NOTE: Handling of Encoding::UndefinedConversionError is added as a quick fix to
+        #       the issue between Ruby encoded strings and libddwaf produced events and now
+        #       is under investigation.
+        def json_parse(value)
+          JSON.dump(value)
+        rescue ArgumentError, Encoding::UndefinedConversionError, JSON::JSONError => e
+          AppSec.telemetry.report(e, description: 'AppSec: Failed to convert value into JSON')
+
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/ext.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Ext
+      RASP_SQLI = 'sql_injection'
+      RASP_LFI = 'lfi'
+      RASP_SSRF = 'ssrf'
+
+      PRODUCT_BIT = 0b00000010
+
+      INTERRUPT = :datadog_appsec_interrupt
+      CONTEXT_KEY = 'datadog.appsec.context'
+      ACTIVE_CONTEXT_KEY = :datadog_appsec_active_context
+      EXPLOIT_PREVENTION_EVENT_CATEGORY = 'exploit'
+
+      TAG_APPSEC_ENABLED = '_dd.appsec.enabled'
+      TAG_METASTRUCT_STACK_TRACE = '_dd.stack'
+
+      TELEMETRY_METRICS_NAMESPACE = 'appsec'
+    end
+  end
+end

data/lib/datadog/appsec/extensions.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require_relative 'configuration'
+require_relative '../core/configuration'
+
+module Datadog
+  module AppSec
+    # Extends Datadog tracing with AppSec features
+    module Extensions
+      # Inject AppSec into global objects.
+      def self.activate!
+        Core::Configuration::Settings.extend(Configuration::Settings)
+      end
+    end
+  end
+end

data/lib/datadog/appsec/instrumentation/gateway/argument.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Instrumentation
+      class Gateway
+        # Base class for Gateway Arguments
+        class Argument; end # rubocop:disable Lint/EmptyClass
+
+        # Gateway User argument
+        # NOTE: This class is a subject of elimination and will be removed when
+        #       the event system is refactored.
+        class User < Argument
+          attr_reader :id, :login, :session_id
+
+          def initialize(id = nil, login = nil, session_id = nil)
+            super()
+
+            @id = id
+            @login = login
+            @session_id = session_id
+          end
+        end
+
+        # This class is used to pass arbitrary data to the event system with an
+        # option to tie it to a context.
+        #
+        # NOTE: This class is a subject of elimination and will be removed when
+        #       the event system is refactored.
+        class DataContainer < Argument
+          attr_reader :data, :context
+
+          def initialize(data, context:)
+            super()
+
+            @data = data
+            @context = context
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/instrumentation/gateway/middleware.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Instrumentation
+      class Gateway
+        # NOTE: This class extracted as-is and will be deprecated
+        # Instrumentation gateway middleware
+        class Middleware
+          attr_reader :key, :block
+
+          def initialize(key, &block)
+            @key = key
+            @block = block
+          end
+
+          def call(stack, env)
+            @block.call(stack, env)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/instrumentation/gateway.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require_relative 'gateway/middleware'
+
+module Datadog
+  module AppSec
+    # Instrumentation for AppSec
+    module Instrumentation
+      # Instrumentation gateway implementation
+      class Gateway
+        def initialize
+          @middlewares = Hash.new { |h, k| h[k] = [] }
+          @pushed_events = {}
+        end
+
+        # NOTE: Be careful with pushed names because every pushed event name
+        #       is recorded in order to provide an ability to any subscriber
+        #       to check wether an arbitrary event had happened.
+        #
+        # WARNING: If we start pushing generated names we should consider
+        #          limiting the storage of pushed names.
+        def push(name, env, &block)
+          @pushed_events[name] = true
+
+          block ||= -> {}
+          middlewares_for_name = @middlewares[name]
+
+          return [block.call, nil] if middlewares_for_name.empty?
+
+          wrapped = lambda do |_env|
+            [block.call, nil]
+          end
+
+          # TODO: handle exceptions, except for wrapped
+          stack = middlewares_for_name.reverse.reduce(wrapped) do |next_, middleware|
+            lambda do |env_|
+              middleware.call(next_, env_)
+            end
+          end
+
+          stack.call(env)
+        end
+
+        def watch(name, key, &block)
+          @middlewares[name] << Middleware.new(key, &block) unless @middlewares[name].any? { |m| m.key == key }
+        end
+
+        def pushed?(name)
+          @pushed_events.key?(name)
+        end
+      end
+
+      # NOTE: This left as-is and will be depricated soon.
+      def self.gateway
+        @gateway ||= Gateway.new # TODO: not thread safe
+      end
+    end
+  end
+end

data/lib/datadog/appsec/instrumentation.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    # Instrumentation for AppSec
+    module Instrumentation
+    end
+  end
+end

data/lib/datadog/appsec/metrics/collector.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Metrics
+      # A class responsible for collecting WAF and RASP call metrics.
+      class Collector
+        Store = Struct.new(
+          :evals,
+          :matches,
+          :errors,
+          :timeouts,
+          :duration_ns,
+          :duration_ext_ns,
+          :inputs_truncated,
+          keyword_init: true
+        )
+
+        attr_reader :waf, :rasp
+
+        def initialize
+          @mutex = Mutex.new
+
+          @waf = Store.new(
+            evals: 0, matches: 0, errors: 0, timeouts: 0, duration_ns: 0, duration_ext_ns: 0, inputs_truncated: 0
+          )
+          @rasp = Store.new(
+            evals: 0, matches: 0, errors: 0, timeouts: 0, duration_ns: 0, duration_ext_ns: 0, inputs_truncated: 0
+          )
+        end
+
+        def record_waf(result)
+          @mutex.synchronize do
+            @waf.evals += 1
+            @waf.matches += 1 if result.match?
+            @waf.errors += 1 if result.error?
+            @waf.timeouts += 1 if result.timeout?
+            @waf.duration_ns += result.duration_ns
+            @waf.duration_ext_ns += result.duration_ext_ns
+            @waf.inputs_truncated += 1 if result.input_truncated?
+          end
+        end
+
+        def record_rasp(result)
+          @mutex.synchronize do
+            @rasp.evals += 1
+            @waf.matches += 1 if result.match?
+            @waf.errors += 1 if result.error?
+            @rasp.timeouts += 1 if result.timeout?
+            @rasp.duration_ns += result.duration_ns
+            @rasp.duration_ext_ns += result.duration_ext_ns
+            @rasp.inputs_truncated += 1 if result.input_truncated?
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/metrics/exporter.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Metrics
+      # A class responsible for exporting WAF and RASP call metrics.
+      module Exporter
+        module_function
+
+        def export_waf_metrics(metrics, span)
+          return if metrics.evals.zero?
+
+          span.set_tag('_dd.appsec.waf.timeouts', metrics.timeouts)
+          span.set_tag('_dd.appsec.waf.duration', convert_ns_to_us(metrics.duration_ns))
+          span.set_tag('_dd.appsec.waf.duration_ext', convert_ns_to_us(metrics.duration_ext_ns))
+        end
+
+        def export_rasp_metrics(metrics, span)
+          return if metrics.evals.zero?
+
+          span.set_tag('_dd.appsec.rasp.rule.eval', metrics.evals)
+          span.set_tag('_dd.appsec.rasp.timeout', 1) unless metrics.timeouts.zero?
+          span.set_tag('_dd.appsec.rasp.duration', convert_ns_to_us(metrics.duration_ns))
+          span.set_tag('_dd.appsec.rasp.duration_ext', convert_ns_to_us(metrics.duration_ext_ns))
+        end
+
+        # private
+
+        def convert_ns_to_us(value)
+          value / 1000.0
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/metrics/telemetry.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Metrics
+      # A class responsible for reporting WAF and RASP telemetry metrics.
+      module Telemetry
+        module_function
+
+        def report_rasp(type, result)
+          return if result.error?
+
+          tags = {rule_type: type, waf_version: Datadog::AppSec::WAF::VERSION::BASE_STRING}
+          namespace = Ext::TELEMETRY_METRICS_NAMESPACE
+
+          AppSec.telemetry.inc(namespace, 'rasp.rule.eval', 1, tags: tags)
+          AppSec.telemetry.inc(namespace, 'rasp.rule.match', 1, tags: tags) if result.match?
+          AppSec.telemetry.inc(namespace, 'rasp.timeout', 1, tags: tags) if result.timeout?
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/metrics/telemetry_exporter.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Metrics
+      # A class responsible for exporting WAF request metrics via Telemetry.
+      module TelemetryExporter
+        module_function
+
+        def export_waf_request_metrics(metrics, context)
+          AppSec.telemetry.inc(
+            Ext::TELEMETRY_METRICS_NAMESPACE, 'waf.requests', 1,
+            tags: {
+              waf_version: WAF::VERSION::BASE_STRING,
+              event_rules_version: context.waf_runner_ruleset_version,
+              rule_triggered: metrics.matches.positive?.to_s,
+              waf_error: metrics.errors.positive?.to_s,
+              waf_timeout: metrics.timeouts.positive?.to_s,
+              request_blocked: context.interrupted?.to_s,
+              block_failure: 'false',
+              rate_limited: (!context.trace.sampled?).to_s,
+              input_truncated: metrics.inputs_truncated.positive?.to_s,
+            }
+          )
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/metrics.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    # This namespace contains classes related to metrics collection and exportation.
+    module Metrics
+    end
+  end
+end
+
+require_relative 'metrics/collector'
+require_relative 'metrics/exporter'
+require_relative 'metrics/telemetry'
+require_relative 'metrics/telemetry_exporter'

data/lib/datadog/appsec/monitor/gateway/watcher.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require_relative '../../event'
+require_relative '../../security_event'
+require_relative '../../instrumentation/gateway'
+
+module Datadog
+  module AppSec
+    module Monitor
+      module Gateway
+        # Watcher for Apssec internal events
+        module Watcher
+          ARBITRARY_VALUE = 'invalid'
+          EVENT_LOGIN_SUCCESS = 'users.login.success'
+          EVENT_LOGIN_FAILURE = 'users.login.failure'
+          WATCHED_LOGIN_EVENTS = [EVENT_LOGIN_SUCCESS, EVENT_LOGIN_FAILURE].freeze
+
+          class << self
+            def watch
+              gateway = Instrumentation.gateway
+
+              watch_user_id(gateway)
+              watch_user_login(gateway)
+            end
+
+            def watch_user_id(gateway = Instrumentation.gateway)
+              gateway.watch('identity.set_user', :appsec) do |stack, user|
+                context = AppSec.active_context
+
+                if user.id.nil? && user.login.nil? && user.session_id.nil?
+                  Datadog.logger.debug { 'AppSec: skipping WAF check because no user information was provided' }
+                  next stack.call(user)
+                end
+
+                persistent_data = {}
+                persistent_data['usr.id'] = user.id if user.id
+                persistent_data['usr.login'] = user.login if user.login
+                persistent_data['usr.session_id'] = user.session_id if user.session_id
+
+                result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
+
+                if result.match? || result.attributes.any?
+                  context.events.push(
+                    AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
+                  )
+                end
+
+                if result.match?
+                  AppSec::Event.tag(context, result)
+                  AppSec::ActionsHandler.handle(result.actions)
+                end
+
+                stack.call(user)
+              end
+            end
+
+            def watch_user_login(gateway = Instrumentation.gateway)
+              gateway.watch('appsec.events.user_lifecycle', :appsec) do |stack, kind|
+                context = AppSec.active_context
+
+                next stack.call(kind) unless WATCHED_LOGIN_EVENTS.include?(kind)
+
+                persistent_data = {"server.business_logic.#{kind}" => ARBITRARY_VALUE}
+                result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
+
+                if result.match? || result.attributes.any?
+                  context.events.push(
+                    AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
+                  )
+                end
+
+                if result.match?
+                  AppSec::Event.tag(context, result)
+                  AppSec::ActionsHandler.handle(result.actions)
+                end
+
+                stack.call(kind)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/monitor.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require_relative 'monitor/gateway/watcher'
+
+module Datadog
+  module AppSec
+    # Monitor for internal AppSec Events
+    module Monitor
+    end
+  end
+end