datadog 2.8.0 → 2.39.0

data/lib/datadog/appsec/event.rb

@@ -1,193 +1,121 @@
 # frozen_string_literal: true
 
-require 'json'
-require 'zlib'
-
-require_relative 'rate_limiter'
-require_relative '../core/utils/base64'
+require "json"
+require_relative "rate_limiter"
+require_relative "trace_keeper"
+require_relative "compressed_json"
 
 module Datadog
   module AppSec
     # AppSec event
     module Event
+      ATTRIBUTES_SCHEMA_KEY_PREFIX = "_dd.appsec.s."
+      ATTRIBUTES_SCHEMA_MAX_COMPRESSED_SIZE = 25000
       ALLOWED_REQUEST_HEADERS = %w[
-        X-Forwarded-For
-        X-Client-IP
-        X-Real-IP
-        X-Forwarded
-        X-Cluster-Client-IP
-        Forwarded-For
-        Forwarded
-        Via
-        True-Client-IP
-        Content-Length
-        Content-Type
-        Content-Encoding
-        Content-Language
-        Host
-        User-Agent
-        Accept
-        Accept-Encoding
-        Accept-Language
-      ].map!(&:downcase).freeze
-
-      ALLOWED_RESPONSE_HEADERS = %w[
-        Content-Length
-        Content-Type
-        Content-Encoding
-        Content-Language
-      ].map!(&:downcase).freeze
-
-      MAX_ENCODED_SCHEMA_SIZE = 25000
-      # For more information about this number
-      # please check https://github.com/DataDog/dd-trace-rb/pull/3177#issuecomment-1747221082
-      MIN_SCHEMA_SIZE_FOR_COMPRESSION = 260
-
-      # Record events for a trace
-      #
-      # This is expected to be called only once per trace for the rate limiter
-      # to properly apply
+        x-forwarded-for
+        x-client-ip
+        x-real-ip
+        x-forwarded
+        x-cluster-client-ip
+        forwarded-for
+        forwarded
+        via
+        true-client-ip
+        content-length
+        content-type
+        content-encoding
+        content-language
+        host
+        user-agent
+        accept
+        accept-encoding
+        accept-language
+      ].freeze
+
       class << self
-        def record(span, *events)
-          # ensure rate limiter is called only when there are events to record
-          return if events.empty? || span.nil?
+        def tag(context, waf_result)
+          return if context.span.nil?
 
-          Datadog::AppSec::RateLimiter.thread_local.limit do
-            record_via_span(span, *events)
+          if waf_result.actions.key?("block_request") || waf_result.actions.key?("redirect_request")
+            context.span.set_tag("appsec.blocked", "true")
           end
+
+          context.span.set_tag("appsec.event", "true")
         end
 
-        def record_via_span(span, *events)
-          events.group_by { |e| e[:trace] }.each do |trace, event_group|
-            unless trace
-              Datadog.logger.debug { "{ error: 'no trace: cannot record', event_group: #{event_group.inspect}}" }
-              next
-            end
+        def record(context, request: nil)
+          return if context.events.empty? || context.span.nil?
 
-            trace.keep!
-            trace.set_tag(
-              Datadog::Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER,
-              Datadog::Tracing::Sampling::Ext::Decision::ASM
-            )
+          Datadog::AppSec::RateLimiter.thread_local.limit do
+            context.events.group_by(&:trace).each do |trace, event_group|
+              unless trace
+                next Datadog.logger.debug do
+                  "AppSec: Cannot record event group with #{event_group.count} events because it has no trace"
+                end
+              end
 
-            # prepare and gather tags to apply
-            service_entry_tags = build_service_entry_tags(event_group)
+              if event_group.any? { |event| event.keep? || event.schema? }
+                TraceKeeper.keep!(trace)
+
+                context.span["_dd.origin"] = "appsec"
+                context.span.set_tags(request_tags(request)) if request
+              end
 
-            # apply tags to service entry span
-            service_entry_tags.each do |key, value|
-              span.set_tag(key, value)
+              context.span.set_tags(waf_tags(event_group))
             end
           end
         end
 
-        # rubocop:disable Metrics/MethodLength
-        def build_service_entry_tags(event_group)
-          waf_events = []
-          entry_tags = event_group.each_with_object({ '_dd.origin' => 'appsec' }) do |event, tags|
-            # TODO: assume HTTP request context for now
-            if (request = event[:request])
-              request.headers.each do |header, value|
-                tags["http.request.headers.#{header}"] = value if ALLOWED_REQUEST_HEADERS.include?(header.downcase)
-              end
+        private
 
-              tags['http.host'] = request.host
-              tags['http.useragent'] = request.user_agent
-              tags['network.client.ip'] = request.remote_addr
-            end
+        def request_tags(request)
+          tags = {}
 
-            if (response = event[:response])
-              response.headers.each do |header, value|
-                tags["http.response.headers.#{header}"] = value if ALLOWED_RESPONSE_HEADERS.include?(header.downcase)
-              end
-            end
+          tags["http.host"] = request.host if request.host
+          tags["http.useragent"] = request.user_agent if request.user_agent
+          tags["network.client.ip"] = request.remote_addr if request.remote_addr
 
-            waf_result = event[:waf_result]
-            # accumulate triggers
-            waf_events += waf_result.events
+          request.headers.each_with_object(tags) do |(name, value), memo|
+            next unless ALLOWED_REQUEST_HEADERS.include?(name)
 
-            waf_result.derivatives.each do |key, value|
-              parsed_value = json_parse(value)
-              next unless parsed_value
+            memo["http.request.headers.#{name}"] = value
+          end
+        end
 
-              parsed_value_size = parsed_value.size
+        def waf_tags(security_events)
+          triggers = []
 
-              schema_value = if parsed_value_size >= MIN_SCHEMA_SIZE_FOR_COMPRESSION
-                               compressed_and_base64_encoded(parsed_value)
-                             else
-                               parsed_value
-                             end
-              next unless schema_value
+          tags = security_events.each_with_object({}) do |security_event, memo|
+            triggers.concat(security_event.waf_result.events)
 
-              if schema_value.size >= MAX_ENCODED_SCHEMA_SIZE
-                Datadog.logger.debug do
-                  "Schema key: #{key} exceeds the max size value. It will not be included as part of the span tags"
-                end
+            security_event.waf_result.attributes.each do |key, value|
+              next memo[key] = value unless key.start_with?(ATTRIBUTES_SCHEMA_KEY_PREFIX)
+
+              value = CompressedJson.dump(value)
+              next if value.nil?
+
+              if value.size >= ATTRIBUTES_SCHEMA_MAX_COMPRESSED_SIZE
+                Datadog.logger.debug { "AppSec: Schema key '#{key}' will not be included into span tags due to it's size" }
                 next
               end
 
-              tags[key] = schema_value
+              memo[key] = value
             end
-
-            tags
-          end
-
-          appsec_events = json_parse({ triggers: waf_events })
-          entry_tags['_dd.appsec.json'] = appsec_events if appsec_events
-          entry_tags
-        end
-        # rubocop:enable Metrics/MethodLength
-
-        def tag_and_keep!(scope, waf_result)
-          # We want to keep the trace in case of security event
-          scope.trace.keep! if scope.trace
-
-          if scope.service_entry_span
-            scope.service_entry_span.set_tag('appsec.blocked', 'true') if waf_result.actions.key?('block_request')
-            scope.service_entry_span.set_tag('appsec.event', 'true')
           end
 
-          add_distributed_tags(scope.trace)
-        end
-
-        private
-
-        def compressed_and_base64_encoded(value)
-          Datadog::Core::Utils::Base64.strict_encode64(gzip(value))
-        rescue TypeError => e
-          Datadog.logger.debug do
-            "Failed to compress and encode value when populating AppSec::Event. Error: #{e.message}"
-          end
-          nil
+          tags["_dd.appsec.json"] = json_parse({triggers: triggers}) unless triggers.empty?
+          tags
         end
 
+        # NOTE: Handling of Encoding::UndefinedConversionError is added as a quick fix to
+        #       the issue between Ruby encoded strings and libddwaf produced events and now
+        #       is under investigation.
         def json_parse(value)
           JSON.dump(value)
-        rescue ArgumentError => e
-          Datadog.logger.debug do
-            "Failed to parse value to JSON when populating AppSec::Event. Error: #{e.message}"
-          end
-          nil
-        end
+        rescue ArgumentError, Encoding::UndefinedConversionError, JSON::JSONError => e
+          AppSec.telemetry.report(e, description: "AppSec: Failed to convert value into JSON")
 
-        def gzip(value)
-          sio = StringIO.new
-          # For an in depth comparison of Zlib options check https://github.com/DataDog/dd-trace-rb/pull/3177#issuecomment-1747215473
-          gz = Zlib::GzipWriter.new(sio, Zlib::BEST_SPEED, Zlib::DEFAULT_STRATEGY)
-          gz.write(value)
-          gz.close
-          sio.string
-        end
-
-        # Propagate to downstream services the information that the current distributed trace is
-        # containing at least one ASM security event.
-        def add_distributed_tags(trace)
-          return unless trace
-
-          trace.set_tag(
-            Datadog::Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER,
-            Datadog::Tracing::Sampling::Ext::Decision::ASM
-          )
-          trace.set_tag(Datadog::AppSec::Ext::TAG_DISTRIBUTED_APPSEC_EVENT, '1')
+          nil
         end
       end
     end

data/lib/datadog/appsec/ext.rb

@@ -3,12 +3,24 @@
 module Datadog
   module AppSec
     module Ext
+      RASP_SQLI = "sql_injection"
+      RASP_LFI = "lfi"
+      RASP_SSRF = "ssrf"
+      RASP_REQUEST_PHASE = "request"
+      RASP_RESPONSE_PHASE = "response"
+
+      PRODUCT_BIT = 0b00000010
+
       INTERRUPT = :datadog_appsec_interrupt
-      SCOPE_KEY = 'datadog.appsec.scope'
+      CONTEXT_KEY = "datadog.appsec.context"
+      ACTIVE_CONTEXT_KEY = :datadog_appsec_active_context
+      EXPLOIT_PREVENTION_EVENT_CATEGORY = "exploit"
+
+      TAG_APPSEC_ENABLED = "_dd.appsec.enabled"
+      TAG_NORMALIZED_ROUTE = "_dd.appsec.normalized_route"
+      TAG_METASTRUCT_STACK_TRACE = "_dd.stack"
 
-      TAG_APPSEC_ENABLED = '_dd.appsec.enabled'
-      TAG_APM_ENABLED = '_dd.apm.enabled'
-      TAG_DISTRIBUTED_APPSEC_EVENT = '_dd.p.appsec'
+      TELEMETRY_METRICS_NAMESPACE = "appsec"
     end
   end
 end

data/lib/datadog/appsec/extensions.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require_relative 'configuration'
-require_relative '../core/configuration'
+require_relative "configuration"
+require_relative "../core/configuration"
 
 module Datadog
   module AppSec

data/lib/datadog/appsec/instrumentation/gateway/argument.rb

@@ -8,12 +8,33 @@ module Datadog
         class Argument; end # rubocop:disable Lint/EmptyClass
 
         # Gateway User argument
+        # NOTE: This class is a subject of elimination and will be removed when
+        #       the event system is refactored.
         class User < Argument
-          attr_reader :id
+          attr_reader :id, :login, :session_id
 
-          def initialize(id)
+          def initialize(id = nil, login = nil, session_id = nil)
             super()
+
             @id = id
+            @login = login
+            @session_id = session_id
+          end
+        end
+
+        # This class is used to pass arbitrary data to the event system with an
+        # option to tie it to a context.
+        #
+        # NOTE: This class is a subject of elimination and will be removed when
+        #       the event system is refactored.
+        class DataContainer < Argument
+          attr_reader :data, :context
+
+          def initialize(data, context:)
+            super()
+
+            @data = data
+            @context = context
           end
         end
       end

data/lib/datadog/appsec/instrumentation/gateway/middleware.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Instrumentation
+      class Gateway
+        # NOTE: This class extracted as-is and will be deprecated
+        # Instrumentation gateway middleware
+        class Middleware
+          attr_reader :block
+
+          def initialize(&block)
+            @block = block
+          end
+
+          def call(stack, env)
+            @block.call(stack, env)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/instrumentation/gateway.rb

@@ -1,35 +1,20 @@
 # frozen_string_literal: true
 
+require_relative "gateway/middleware"
+
 module Datadog
   module AppSec
     # Instrumentation for AppSec
     module Instrumentation
       # Instrumentation gateway implementation
       class Gateway
-        # Instrumentation gateway middleware
-        class Middleware
-          attr_reader :key, :block
-
-          def initialize(key, &block)
-            @key = key
-            @block = block
-          end
-
-          def call(stack, env)
-            @block.call(stack, env)
-          end
-        end
-
-        private_constant :Middleware
-
         def initialize
           @middlewares = Hash.new { |h, k| h[k] = [] }
         end
 
         def push(name, env, &block)
           block ||= -> {}
-
-          middlewares_for_name = middlewares[name]
+          middlewares_for_name = @middlewares[name]
 
           return [block.call, nil] if middlewares_for_name.empty?
 
@@ -47,15 +32,12 @@ module Datadog
           stack.call(env)
         end
 
-        def watch(name, key, &block)
-          @middlewares[name] << Middleware.new(key, &block) unless middlewares[name].any? { |m| m.key == key }
+        def watch(name, &block)
+          @middlewares[name] << Middleware.new(&block)
         end
-
-        private
-
-        attr_reader :middlewares
       end
 
+      # NOTE: This left as-is and will be depricated soon.
       def self.gateway
         @gateway ||= Gateway.new # TODO: not thread safe
       end

data/lib/datadog/appsec/metrics/collector.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Metrics
+      # A class responsible for collecting WAF and RASP call metrics.
+      class Collector
+        Store = Struct.new(
+          :evals,
+          :matches,
+          :errors,
+          :timeouts,
+          :duration_ns,
+          :duration_ext_ns,
+          :inputs_truncated,
+          :downstream_requests,
+          keyword_init: true
+        )
+
+        DownstreamResponseStore = Struct.new(
+          :content_type_invalid,
+          :content_length_missing,
+          :content_length_too_big,
+          :content_exceed_content_length,
+          keyword_init: true
+        )
+
+        attr_reader :waf, :rasp, :downstream_responses
+
+        def initialize
+          @mutex = Mutex.new
+
+          @waf = Store.new(
+            evals: 0, matches: 0, errors: 0, timeouts: 0, duration_ns: 0,
+            duration_ext_ns: 0, inputs_truncated: 0, downstream_requests: 0
+          )
+
+          @rasp = Store.new(
+            evals: 0, matches: 0, errors: 0, timeouts: 0, duration_ns: 0,
+            duration_ext_ns: 0, inputs_truncated: 0, downstream_requests: 0
+          )
+
+          @downstream_responses = DownstreamResponseStore.new(
+            content_type_invalid: 0, content_length_missing: 0,
+            content_length_too_big: 0, content_exceed_content_length: 0
+          )
+        end
+
+        def record_waf(result)
+          @mutex.synchronize do
+            @waf.evals += 1
+            @waf.matches += 1 if result.match?
+            @waf.errors += 1 if result.error?
+            @waf.timeouts += 1 if result.timeout?
+            @waf.duration_ns += result.duration_ns
+            @waf.duration_ext_ns += result.duration_ext_ns
+            @waf.inputs_truncated += 1 if result.input_truncated?
+          end
+        end
+
+        def record_rasp(result, type:, phase: nil)
+          @mutex.synchronize do
+            @rasp.evals += 1
+            @waf.matches += 1 if result.match?
+            @waf.errors += 1 if result.error?
+            @rasp.timeouts += 1 if result.timeout?
+            @rasp.duration_ns += result.duration_ns
+            @rasp.duration_ext_ns += result.duration_ext_ns
+            @rasp.inputs_truncated += 1 if result.input_truncated?
+            @rasp.downstream_requests += 1 if type == Ext::RASP_SSRF && phase == Ext::RASP_REQUEST_PHASE
+          end
+        end
+
+        def record_ignored_downstream_response_body(reason)
+          @mutex.synchronize { @downstream_responses[reason] += 1 }
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/metrics/exporter.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Metrics
+      # A class responsible for exporting WAF and RASP call metrics
+      module Exporter
+        module_function
+
+        def export_waf_metrics(metrics, span)
+          return if metrics.evals.zero?
+
+          span.set_tag("_dd.appsec.waf.timeouts", metrics.timeouts)
+          span.set_tag("_dd.appsec.waf.duration", convert_ns_to_us(metrics.duration_ns))
+          span.set_tag("_dd.appsec.waf.duration_ext", convert_ns_to_us(metrics.duration_ext_ns))
+        end
+
+        def export_rasp_metrics(metrics, span)
+          return if metrics.evals.zero?
+
+          span.set_tag("_dd.appsec.rasp.rule.eval", metrics.evals)
+          span.set_tag("_dd.appsec.rasp.timeout", 1) unless metrics.timeouts.zero?
+          span.set_tag("_dd.appsec.rasp.duration", convert_ns_to_us(metrics.duration_ns))
+          span.set_tag("_dd.appsec.rasp.duration_ext", convert_ns_to_us(metrics.duration_ext_ns))
+
+          # NOTE: In case of downstream requests being analyzed additionally
+          #       with `Context.run_waf` method, we would need to share it
+          #       between two exporting methods
+          unless metrics.downstream_requests.zero?
+            span.set_tag("_dd.appsec.downstream_request", metrics.downstream_requests)
+          end
+        end
+
+        def export_downstream_response_metrics(metrics, span)
+          metrics.each_pair do |reason, count|
+            next if count.zero?
+
+            span.set_tag("_dd.appsec.downstream_request.response_body_ignored.#{reason}", count)
+          end
+        end
+
+        private_class_method def convert_ns_to_us(value)
+          value / 1000.0
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/metrics/telemetry.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Metrics
+      # A class responsible for reporting WAF and RASP telemetry metrics.
+      module Telemetry
+        ACTION_BLOCK = "block_request"
+        ACTION_REDIRECT = "redirect_request"
+        BLOCK_SUCCESS = "success"
+        BLOCK_IRRELEVANT = "irrelevant"
+
+        module_function
+
+        def report_rasp(type, result, phase: nil)
+          return if result.error?
+
+          tags = {rule_type: type, waf_version: WAF::VERSION::BASE_STRING}
+          tags[:rule_variant] = phase if phase
+
+          context = AppSec.active_context
+          tags[:event_rules_version] = context.waf_runner_ruleset_version if context
+
+          namespace = Ext::TELEMETRY_METRICS_NAMESPACE
+
+          AppSec.telemetry.inc(namespace, "rasp.rule.eval", 1, tags: tags)
+          AppSec.telemetry.inc(namespace, "rasp.timeout", 1, tags: tags) if result.timeout?
+
+          if result.match?
+            blocked = result.actions.key?(ACTION_BLOCK) || result.actions.key?(ACTION_REDIRECT)
+            # NOTE: Mutates tags to avoid an extra hash allocation. Keep this the last .inc call.
+            tags[:block] = blocked ? BLOCK_SUCCESS : BLOCK_IRRELEVANT
+
+            AppSec.telemetry.inc(namespace, "rasp.rule.match", 1, tags: tags)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/metrics/telemetry_exporter.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Metrics
+      # A class responsible for exporting WAF request metrics via Telemetry.
+      module TelemetryExporter
+        module_function
+
+        def export_waf_request_metrics(metrics, context)
+          AppSec.telemetry.inc(
+            Ext::TELEMETRY_METRICS_NAMESPACE, "waf.requests", 1,
+            tags: {
+              waf_version: WAF::VERSION::BASE_STRING,
+              event_rules_version: context.waf_runner_ruleset_version,
+              rule_triggered: metrics.matches.positive?.to_s,
+              waf_error: metrics.errors.positive?.to_s,
+              waf_timeout: metrics.timeouts.positive?.to_s,
+              request_blocked: context.interrupted?.to_s,
+              block_failure: "false",
+              rate_limited: (!context.trace.sampled?).to_s,
+              input_truncated: metrics.inputs_truncated.positive?.to_s,
+            }
+          )
+        end
+
+        def export_api_security_metrics(context)
+          web_framework = context.state[:web_framework]
+          return unless web_framework
+
+          if context.span&.get_tag(Tracing::Metadata::Ext::HTTP::TAG_ROUTE).nil?
+            AppSec.telemetry.inc(
+              AppSec::Ext::TELEMETRY_METRICS_NAMESPACE, "api_security.missing_route", 1,
+              tags: {framework: web_framework}
+            )
+          end
+
+          metric_name = context.state[:schema_extracted] ? "schema" : "no_schema"
+          AppSec.telemetry.inc(
+            AppSec::Ext::TELEMETRY_METRICS_NAMESPACE, "api_security.request.#{metric_name}", 1,
+            tags: {framework: web_framework}
+          )
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/metrics.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require_relative "metrics/collector"
+require_relative "metrics/exporter"
+require_relative "metrics/telemetry"
+require_relative "metrics/telemetry_exporter"
+
+module Datadog
+  module AppSec
+    # This namespace contains classes related to metrics collection and exportation.
+    module Metrics
+    end
+  end
+end