datadog 2.21.0 → 2.23.0

data/lib/datadog/appsec/contrib/rails/gateway/watcher.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative '../../../event'
+require_relative '../../../trace_keeper'
 require_relative '../../../security_event'
 require_relative '../../../instrumentation/gateway'
 
@@ -35,7 +36,9 @@ module Datadog
                       AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
                     )
 
-                    AppSec::Event.tag_and_keep!(context, result)
+                    AppSec::Event.tag(context, result)
+                    TraceKeeper.keep!(context.trace) if result.keep?
+
                     AppSec::ActionsHandler.handle(result.actions)
                   end
 
@@ -57,7 +60,9 @@ module Datadog
                       AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
                     )
 
-                    AppSec::Event.tag_and_keep!(context, result)
+                    AppSec::Event.tag(context, result)
+                    TraceKeeper.keep!(context.trace) if result.keep?
+
                     AppSec::ActionsHandler.handle(result.actions)
                   end
 

data/lib/datadog/appsec/contrib/rails/patcher.rb

@@ -10,6 +10,7 @@ require_relative 'gateway/watcher'
 require_relative 'gateway/request'
 require_relative 'patches/render_to_body_patch'
 require_relative 'patches/process_action_patch'
+require_relative '../../api_security/endpoint_collection/rails_collector'
 
 require_relative '../../../tracing/contrib/rack/middlewares'
 
@@ -20,6 +21,7 @@ module Datadog
         # Patcher for AppSec on Rails
         module Patcher
           GUARD_ACTION_CONTROLLER_ONCE_PER_APP = Hash.new { |h, key| h[key] = Datadog::Core::Utils::OnlyOnce.new }
+          GUARD_ROUTES_REPORTING_ONCE_PER_APP = Hash.new { |h, key| h[key] = Datadog::Core::Utils::OnlyOnce.new }
           BEFORE_INITIALIZE_ONLY_ONCE_PER_APP = Hash.new { |h, key| h[key] = Datadog::Core::Utils::OnlyOnce.new }
           AFTER_INITIALIZE_ONLY_ONCE_PER_APP = Hash.new { |h, key| h[key] = Datadog::Core::Utils::OnlyOnce.new }
 
@@ -38,6 +40,7 @@ module Datadog
             patch_before_initialize
             patch_after_initialize
             patch_action_controller
+            subscribe_to_routes_loaded
 
             Patcher.instance_variable_set(:@patched, true)
           end
@@ -128,7 +131,34 @@ module Datadog
               GUARD_ACTION_CONTROLLER_ONCE_PER_APP[self].run do
                 ::ActionController::Base.prepend(Patches::RenderToBodyPatch)
               end
+
+              # Rails 7.1 adds `after_routes_loaded` hook
+              if Datadog::AppSec::Contrib::Rails::Patcher.target_version < Gem::Version.new('7.1')
+                Datadog::AppSec::Contrib::Rails::Patcher.report_routes_via_telemetry(::Rails.application.routes.routes)
+              end
+            end
+          end
+
+          def subscribe_to_routes_loaded
+            ::ActiveSupport.on_load(:after_routes_loaded) do |app|
+              Datadog::AppSec::Contrib::Rails::Patcher.report_routes_via_telemetry(app.routes.routes)
+            end
+          end
+
+          def report_routes_via_telemetry(routes)
+            # We do not support Rails 4.x for Endpoint Collection,
+            # mainly because the Route#verb was a Regexp before Rails 5.0
+            return if target_version < Gem::Version.new('5.0')
+            return unless Datadog.configuration.appsec.api_security.endpoint_collection.enabled
+            return unless AppSec.telemetry
+
+            GUARD_ROUTES_REPORTING_ONCE_PER_APP[::Rails.application].run do
+              AppSec.telemetry.app_endpoints_loaded(
+                APISecurity::EndpointCollection::RailsCollector.new(routes).to_enum
+              )
             end
+          rescue => e
+            AppSec.telemetry&.report(e, description: 'failed to report application endpoints')
           end
 
           def setup_security

data/lib/datadog/appsec/contrib/rest_client/request_ssrf_detection_patch.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative '../../event'
+require_relative '../../trace_keeper'
 require_relative '../../security_event'
 
 module Datadog
@@ -18,7 +19,8 @@ module Datadog
             result = context.run_rasp(Ext::RASP_SSRF, {}, ephemeral_data, Datadog.configuration.appsec.waf_timeout)
 
             if result.match?
-              AppSec::Event.tag_and_keep!(context, result)
+              AppSec::Event.tag(context, result)
+              TraceKeeper.keep!(context.trace) if result.keep?
 
               context.events.push(
                 AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)

data/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative '../../../event'
+require_relative '../../../trace_keeper'
 require_relative '../../../security_event'
 require_relative '../../../instrumentation/gateway'
 
@@ -30,14 +31,16 @@ module Datadog
 
                   result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
 
-                  if result.match? || !result.derivatives.empty?
+                  if result.match? || !result.attributes.empty?
                     context.events.push(
                       AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
                     )
                   end
 
                   if result.match?
-                    AppSec::Event.tag_and_keep!(context, result)
+                    AppSec::Event.tag(context, result)
+                    TraceKeeper.keep!(context.trace) if result.keep?
+
                     AppSec::ActionsHandler.handle(result.actions)
                   end
 
@@ -56,7 +59,8 @@ module Datadog
                   result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
 
                   if result.match?
-                    AppSec::Event.tag_and_keep!(context, result)
+                    AppSec::Event.tag(context, result)
+                    TraceKeeper.keep!(context.trace) if result.keep?
 
                     context.events.push(
                       AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
@@ -83,7 +87,9 @@ module Datadog
                       AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
                     )
 
-                    AppSec::Event.tag_and_keep!(context, result)
+                    AppSec::Event.tag(context, result)
+                    TraceKeeper.keep!(context.trace) if result.keep?
+
                     AppSec::ActionsHandler.handle(result.actions)
                   end
 

data/lib/datadog/appsec/event.rb

@@ -9,8 +9,8 @@ module Datadog
   module AppSec
     # AppSec event
     module Event
-      DERIVATIVE_SCHEMA_KEY_PREFIX = '_dd.appsec.s.'
-      DERIVATIVE_SCHEMA_MAX_COMPRESSED_SIZE = 25000
+      ATTRIBUTES_SCHEMA_KEY_PREFIX = '_dd.appsec.s.'
+      ATTRIBUTES_SCHEMA_MAX_COMPRESSED_SIZE = 25000
       ALLOWED_REQUEST_HEADERS = %w[
         x-forwarded-for
         x-client-ip
@@ -40,16 +40,14 @@ module Datadog
       ].freeze
 
       class << self
-        def tag_and_keep!(context, waf_result)
-          TraceKeeper.keep!(context.trace)
+        def tag(context, waf_result)
+          return if context.span.nil?
 
-          if context.span
-            if waf_result.actions.key?('block_request') || waf_result.actions.key?('redirect_request')
-              context.span.set_tag('appsec.blocked', 'true')
-            end
-
-            context.span.set_tag('appsec.event', 'true')
+          if waf_result.actions.key?('block_request') || waf_result.actions.key?('redirect_request')
+            context.span.set_tag('appsec.blocked', 'true')
           end
+
+          context.span.set_tag('appsec.event', 'true')
         end
 
         def record(context, request: nil, response: nil)
@@ -63,7 +61,7 @@ module Datadog
                 end
               end
 
-              if event_group.any? { |event| event.attack? || event.schema? }
+              if event_group.any? { |event| event.keep? || event.schema? }
                 TraceKeeper.keep!(trace)
 
                 context.span['_dd.origin'] = 'appsec'
@@ -106,13 +104,13 @@ module Datadog
           tags = security_events.each_with_object({}) do |security_event, memo|
             triggers.concat(security_event.waf_result.events)
 
-            security_event.waf_result.derivatives.each do |key, value|
-              next memo[key] = value unless key.start_with?(DERIVATIVE_SCHEMA_KEY_PREFIX)
+            security_event.waf_result.attributes.each do |key, value|
+              next memo[key] = value unless key.start_with?(ATTRIBUTES_SCHEMA_KEY_PREFIX)
 
               value = CompressedJson.dump(value)
               next if value.nil?
 
-              if value.size >= DERIVATIVE_SCHEMA_MAX_COMPRESSED_SIZE
+              if value.size >= ATTRIBUTES_SCHEMA_MAX_COMPRESSED_SIZE
                 Datadog.logger.debug { "AppSec: Schema key '#{key}' will not be included into span tags due to it's size" }
                 next
               end

data/lib/datadog/appsec/metrics/collector.rb

@@ -5,14 +5,28 @@ module Datadog
     module Metrics
       # A class responsible for collecting WAF and RASP call metrics.
       class Collector
-        Store = Struct.new(:evals, :matches, :errors, :timeouts, :duration_ns, :duration_ext_ns, keyword_init: true)
+        Store = Struct.new(
+          :evals,
+          :matches,
+          :errors,
+          :timeouts,
+          :duration_ns,
+          :duration_ext_ns,
+          :inputs_truncated,
+          keyword_init: true
+        )
 
         attr_reader :waf, :rasp
 
         def initialize
           @mutex = Mutex.new
-          @waf = Store.new(evals: 0, matches: 0, errors: 0, timeouts: 0, duration_ns: 0, duration_ext_ns: 0)
-          @rasp = Store.new(evals: 0, matches: 0, errors: 0, timeouts: 0, duration_ns: 0, duration_ext_ns: 0)
+
+          @waf = Store.new(
+            evals: 0, matches: 0, errors: 0, timeouts: 0, duration_ns: 0, duration_ext_ns: 0, inputs_truncated: 0
+          )
+          @rasp = Store.new(
+            evals: 0, matches: 0, errors: 0, timeouts: 0, duration_ns: 0, duration_ext_ns: 0, inputs_truncated: 0
+          )
         end
 
         def record_waf(result)
@@ -23,6 +37,7 @@ module Datadog
             @waf.timeouts += 1 if result.timeout?
             @waf.duration_ns += result.duration_ns
             @waf.duration_ext_ns += result.duration_ext_ns
+            @waf.inputs_truncated += 1 if result.input_truncated?
           end
         end
 
@@ -34,6 +49,7 @@ module Datadog
             @rasp.timeouts += 1 if result.timeout?
             @rasp.duration_ns += result.duration_ns
             @rasp.duration_ext_ns += result.duration_ext_ns
+            @rasp.inputs_truncated += 1 if result.input_truncated?
           end
         end
       end

data/lib/datadog/appsec/metrics/telemetry_exporter.rb

@@ -18,7 +18,8 @@ module Datadog
               waf_timeout: metrics.timeouts.positive?.to_s,
               request_blocked: context.interrupted?.to_s,
               block_failure: 'false',
-              rate_limited: (!context.trace.sampled?).to_s
+              rate_limited: (!context.trace.sampled?).to_s,
+              input_truncated: metrics.inputs_truncated.positive?.to_s,
             }
           )
         end

data/lib/datadog/appsec/monitor/gateway/watcher.rb

@@ -39,14 +39,14 @@ module Datadog
 
                 result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
 
-                if result.match? || result.derivatives.any?
+                if result.match? || result.attributes.any?
                   context.events.push(
                     AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
                   )
                 end
 
                 if result.match?
-                  AppSec::Event.tag_and_keep!(context, result)
+                  AppSec::Event.tag(context, result)
                   AppSec::ActionsHandler.handle(result.actions)
                 end
 
@@ -63,14 +63,14 @@ module Datadog
                 persistent_data = {"server.business_logic.#{kind}" => ARBITRARY_VALUE}
                 result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
 
-                if result.match? || result.derivatives.any?
+                if result.match? || result.attributes.any?
                   context.events.push(
                     AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
                   )
                 end
 
                 if result.match?
-                  AppSec::Event.tag_and_keep!(context, result)
+                  AppSec::Event.tag(context, result)
                   AppSec::ActionsHandler.handle(result.actions)
                 end
 

data/lib/datadog/appsec/remote.rb

@@ -12,19 +12,27 @@ module Datadog
       class NoRulesError < StandardError; end
 
       class << self
-        CAP_ASM_RESERVED_1 = 1 << 0   # RESERVED
-        CAP_ASM_ACTIVATION = 1 << 1   # Remote activation via ASM_FEATURES product
-        CAP_ASM_IP_BLOCKING = 1 << 2   # accept IP blocking data from ASM_DATA product
-        CAP_ASM_DD_RULES = 1 << 3   # read ASM rules from ASM_DD product
-        CAP_ASM_EXCLUSIONS = 1 << 4   # exclusion filters (passlist) via ASM product
-        CAP_ASM_REQUEST_BLOCKING = 1 << 5   # can block on request info
-        CAP_ASM_RESPONSE_BLOCKING = 1 << 6   # can block on response info
-        CAP_ASM_USER_BLOCKING = 1 << 7   # accept user blocking data from ASM_DATA product
-        CAP_ASM_CUSTOM_RULES = 1 << 8   # accept custom rules
-        CAP_ASM_CUSTOM_BLOCKING_RESPONSE = 1 << 9   # supports custom http code or redirect sa blocking response
-        CAP_ASM_TRUSTED_IPS = 1 << 10  # supports trusted ip
-        CAP_ASM_RASP_SSRF = 1 << 23  # support for server-side request forgery exploit prevention rules
-        CAP_ASM_RASP_SQLI = 1 << 21  # support for SQL injection exploit prevention rules
+        CAP_ASM_RESERVED_1 = 1 << 0
+        CAP_ASM_ACTIVATION = 1 << 1
+        CAP_ASM_IP_BLOCKING = 1 << 2
+        CAP_ASM_DD_RULES = 1 << 3
+        CAP_ASM_EXCLUSIONS = 1 << 4
+        CAP_ASM_REQUEST_BLOCKING = 1 << 5
+        CAP_ASM_RESPONSE_BLOCKING = 1 << 6
+        CAP_ASM_USER_BLOCKING = 1 << 7
+        CAP_ASM_CUSTOM_RULES = 1 << 8
+        CAP_ASM_CUSTOM_BLOCKING_RESPONSE = 1 << 9
+        CAP_ASM_TRUSTED_IPS = 1 << 10
+        CAP_ASM_PROCESSOR_OVERRIDES = 1 << 16
+        CAP_ASM_CUSTOM_DATA_SCANNERS = 1 << 17
+        CAP_ASM_RASP_SSRF = 1 << 23
+        CAP_ASM_RASP_SQLI = 1 << 21
+        CAP_ASM_AUTO_USER_INSTRUM_MODE = 1 << 31
+        CAP_ASM_ENDPOINT_FINGERPRINT = 1 << 32
+        CAP_ASM_SESSION_FINGERPRINT = 1 << 33
+        CAP_ASM_NETWORK_FINGERPRINT = 1 << 34
+        CAP_ASM_HEADER_FINGERPRINT = 1 << 35
+        CAP_ASM_TRACE_TAGGING_RULES = 1 << 43
 
         # TODO: we need to dynamically add CAP_ASM_ACTIVATION once we support it
         ASM_CAPABILITIES = [
@@ -37,8 +45,16 @@ module Datadog
           CAP_ASM_CUSTOM_RULES,
           CAP_ASM_CUSTOM_BLOCKING_RESPONSE,
           CAP_ASM_TRUSTED_IPS,
+          CAP_ASM_PROCESSOR_OVERRIDES,
+          CAP_ASM_CUSTOM_DATA_SCANNERS,
           CAP_ASM_RASP_SSRF,
           CAP_ASM_RASP_SQLI,
+          CAP_ASM_AUTO_USER_INSTRUM_MODE,
+          CAP_ASM_ENDPOINT_FINGERPRINT,
+          CAP_ASM_SESSION_FINGERPRINT,
+          CAP_ASM_NETWORK_FINGERPRINT,
+          CAP_ASM_HEADER_FINGERPRINT,
+          CAP_ASM_TRACE_TAGGING_RULES,
         ].freeze
 
         ASM_PRODUCTS = [

data/lib/datadog/appsec/response.rb

@@ -7,6 +7,8 @@ module Datadog
   module AppSec
     # AppSec response
     class Response
+      SECURITY_RESPONSE_ID_PLACEHOLDER = '[security_response_id]'
+
       attr_reader :status, :headers, :body
 
       def initialize(status:, headers: {}, body: [])
@@ -37,16 +39,26 @@ module Datadog
           Response.new(
             status: interrupt_params['status_code']&.to_i || 403,
             headers: {'Content-Type' => content_type},
-            body: [content(content_type)],
+            body: [
+              content(
+                security_response_id: interrupt_params['security_response_id'],
+                content_type: content_type
+              )
+            ],
           )
         end
 
         def redirect_response(interrupt_params)
           status_code = interrupt_params['status_code'].to_i
+          location = interrupt_params.fetch('location')
+
+          if (security_response_id = interrupt_params.fetch('security_response_id'))
+            location.gsub!(SECURITY_RESPONSE_ID_PLACEHOLDER, security_response_id)
+          end
 
           Response.new(
             status: ((status_code >= 300 && status_code < 400) ? status_code : 303),
-            headers: {'Location' => interrupt_params.fetch('location')},
+            headers: {'Location' => location},
             body: [],
           )
         end
@@ -82,16 +94,18 @@ module Datadog
           DEFAULT_CONTENT_TYPE
         end
 
-        def content(content_type)
+        def content(security_response_id:, content_type:)
           content_format = CONTENT_TYPE_TO_FORMAT[content_type]
 
           using_default = Datadog.configuration.appsec.block.templates.using_default?(content_format)
 
-          if using_default
+          template = if using_default
             Datadog::AppSec::Assets.blocked(format: content_format)
           else
             Datadog.configuration.appsec.block.templates.send(content_format)
           end
+
+          template.gsub(SECURITY_RESPONSE_ID_PLACEHOLDER, security_response_id.to_s)
         end
       end
     end

data/lib/datadog/appsec/security_engine/result.rb

@@ -7,20 +7,30 @@ module Datadog
       module Result
         # A generic result without indication of its type.
         class Base
-          attr_reader :events, :actions, :derivatives, :duration_ns, :duration_ext_ns
+          attr_reader :events, :actions, :attributes, :duration_ns, :duration_ext_ns
 
-          def initialize(events:, actions:, derivatives:, timeout:, duration_ns:, duration_ext_ns:)
+          def initialize(events:, actions:, attributes:, duration_ns:, duration_ext_ns:, timeout:, keep:, input_truncated:)
             @events = events
             @actions = actions
-            @derivatives = derivatives
-
-            @timeout = timeout
+            @attributes = attributes
             @duration_ns = duration_ns
             @duration_ext_ns = duration_ext_ns
+
+            @keep = !!keep
+            @timeout = !!timeout
+            @input_truncated = !!input_truncated
           end
 
           def timeout?
-            !!@timeout
+            @timeout
+          end
+
+          def keep?
+            @keep
+          end
+
+          def input_truncated?
+            @input_truncated
           end
 
           def match?
@@ -56,19 +66,28 @@ module Datadog
 
         # A result that indicates an internal security library error
         class Error
-          attr_reader :events, :actions, :derivatives, :duration_ns, :duration_ext_ns
+          attr_reader :events, :actions, :attributes, :duration_ns, :duration_ext_ns
 
-          def initialize(duration_ext_ns:)
+          def initialize(duration_ext_ns:, input_truncated:)
             @events = []
-            @actions = @derivatives = {}
+            @actions = @attributes = {}
             @duration_ns = 0
             @duration_ext_ns = duration_ext_ns
+            @input_truncated = !!input_truncated
+          end
+
+          def keep?
+            false
           end
 
           def timeout?
             false
           end
 
+          def input_truncated?
+            @input_truncated
+          end
+
           def match?
             false
           end

data/lib/datadog/appsec/security_engine/runner.rb

@@ -42,17 +42,19 @@ module Datadog
           report_execution(result)
 
           unless SUCCESSFUL_EXECUTION_CODES.include?(result.status)
-            return Result::Error.new(duration_ext_ns: stop_ns - start_ns)
+            return Result::Error.new(duration_ext_ns: stop_ns - start_ns, input_truncated: result.input_truncated?)
           end
 
           klass = (result.status == :match) ? Result::Match : Result::Ok
           klass.new(
             events: result.events,
             actions: result.actions,
-            derivatives: result.derivatives,
-            timeout: result.timeout,
-            duration_ns: result.total_runtime,
-            duration_ext_ns: (stop_ns - start_ns)
+            attributes: result.attributes,
+            keep: result.keep?,
+            timeout: result.timeout?,
+            duration_ns: result.duration,
+            duration_ext_ns: (stop_ns - start_ns),
+            input_truncated: result.input_truncated?
           )
         ensure
           @mutex.unlock
@@ -80,11 +82,19 @@ module Datadog
           Datadog.logger.debug { "#{@debug_tag} execution error: #{e} backtrace: #{e.backtrace&.first(3)}" }
           AppSec.telemetry.report(e, description: 'libddwaf-rb internal low-level error')
 
-          WAF::Result.new(:err_internal, [], 0, false, [], [])
+          WAF::Result.new(
+            status: :err_internal,
+            events: [],
+            actions: {},
+            attributes: {},
+            duration: 0,
+            keep: false,
+            timeout: false
+          )
         end
 
         def report_execution(result)
-          Datadog.logger.debug { "#{@debug_tag} execution timed out: #{result.inspect}" } if result.timeout
+          Datadog.logger.debug { "#{@debug_tag} execution timed out: #{result.inspect}" } if result.timeout?
 
           if SUCCESSFUL_EXECUTION_CODES.include?(result.status)
             Datadog.logger.debug { "#{@debug_tag} execution result: #{result.inspect}" }

data/lib/datadog/appsec/security_event.rb

@@ -3,7 +3,7 @@
 module Datadog
   module AppSec
     # A class that represents a security event of any kind. It could be an event
-    # representing an attack or fingerprinting results as derivatives or an API
+    # representing an attack or fingerprinting results as attributes or an API
     # security check with extracted schema.
     class SecurityEvent
       SCHEMA_KEY_PREFIX = '_dd.appsec.s.'
@@ -17,22 +17,20 @@ module Datadog
         @span = span
       end
 
-      def attack?
-        return @is_attack if defined?(@is_attack)
-
-        @is_attack = @waf_result.is_a?(SecurityEngine::Result::Match)
+      def keep?
+        @waf_result.keep?
       end
 
       def schema?
         return @has_schema if defined?(@has_schema)
 
-        @has_schema = @waf_result.derivatives.any? { |name, _| name.start_with?(SCHEMA_KEY_PREFIX) }
+        @has_schema = @waf_result.attributes.any? { |name, _| name.start_with?(SCHEMA_KEY_PREFIX) }
       end
 
       def fingerprint?
         return @has_fingerprint if defined?(@has_fingerprint)
 
-        @has_fingerprint = @waf_result.derivatives.any? { |name, _| name.start_with?(FINGERPRINT_KEY_PREFIX) }
+        @has_fingerprint = @waf_result.attributes.any? { |name, _| name.start_with?(FINGERPRINT_KEY_PREFIX) }
       end
     end
   end