ddtrace 1.9.0 → 1.10.0

data/lib/datadog/appsec/assets/waf_rules/strict.json

@@ -1,7 +1,7 @@
 {
   "version": "2.2",
   "metadata": {
-    "rules_version": "1.4.3"
+    "rules_version": "1.5.2"
   },
   "rules": [
     {
@@ -158,6 +158,33 @@
         "lowercase"
       ]
     },
+    {
+      "id": "crs-921-140",
+      "name": "HTTP Header Injection Attack via headers",
+      "tags": {
+        "type": "http_protocol_violation",
+        "crs_id": "921140",
+        "category": "attack_attempt"
+      },
+      "conditions": [
+        {
+          "parameters": {
+            "inputs": [
+              {
+                "address": "server.request.headers.no_cookies"
+              }
+            ],
+            "regex": "[\\n\\r]",
+            "options": {
+              "case_sensitive": true,
+              "min_length": 1
+            }
+          },
+          "operator": "match_regex"
+        }
+      ],
+      "transformers": []
+    },
     {
       "id": "crs-932-100",
       "name": "Remote Command Execution: Unix Command Injection",
@@ -1498,6 +1525,36 @@
         "removeNulls"
       ]
     },
+    {
+      "id": "sqr-000-007",
+      "name": "NoSQL: Detect common exploitation strategy",
+      "tags": {
+        "type": "nosql_injection",
+        "category": "attack_attempt"
+      },
+      "conditions": [
+        {
+          "parameters": {
+            "inputs": [
+              {
+                "address": "server.request.query"
+              },
+              {
+                "address": "server.request.body"
+              },
+              {
+                "address": "server.request.path_params"
+              }
+            ],
+            "regex": "^\\$(eq|ne|(l|g)te?|n?in|not|(n|x|)or|and|regex|where|expr|exists)$"
+          },
+          "operator": "match_regex"
+        }
+      ],
+      "transformers": [
+        "keys_only"
+      ]
+    },
     {
       "id": "sqr-000-011",
       "name": "Node.js: Prototype pollution",
@@ -1530,4 +1587,4 @@
       "transformers": []
     }
   ]
-}
+}

data/lib/datadog/appsec/assets.rb

@@ -1,5 +1,3 @@
-# typed: ignore
-
 require 'pathname'
 
 module Datadog

data/lib/datadog/appsec/autoload.rb

@@ -1,18 +1,11 @@
-# typed: ignore
-
 if %w[1 true].include?((ENV['DD_APPSEC_ENABLED'] || '').downcase)
-  require_relative '../../ddtrace'
-
-  begin
-    require_relative '../appsec'
-  rescue StandardError => e
-    puts "AppSec failed to load. No security check will be performed. error: #{e.class.name} #{e.message}"
-  end
-
   begin
     require_relative 'contrib/auto_instrument'
     Datadog::AppSec::Contrib::AutoInstrument.patch_all
   rescue StandardError => e
-    puts "AppSec failed to instrument. No security check will be performed. error: #{e.class.name} #{e.message}"
+    Kernel.warn(
+      '[ddtrace] AppSec failed to instrument. No security check will be performed. error: ' \
+      " #{e.class.name} #{e.message}"
+    )
   end
 end

data/lib/datadog/appsec/component.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require_relative 'processor'
+
+module Datadog
+  module AppSec
+    # Core-pluggable component for AppSec
+    class Component
+      class << self
+        def build_appsec_component(settings)
+          return unless settings.enabled
+
+          processor = create_processor
+          new(processor: processor)
+        end
+
+        private
+
+        def create_processor
+          processor = Processor.new
+          return nil unless processor.ready?
+
+          processor
+        end
+      end
+
+      attr_reader :processor
+
+      def initialize(processor:)
+        @processor = processor
+      end
+
+      def shutdown!
+        if processor && processor.ready?
+          processor.finalize
+          @processor = nil
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/configuration/settings.rb

@@ -1,5 +1,3 @@
-# typed: ignore
-
 module Datadog
   module AppSec
     module Configuration
@@ -139,7 +137,14 @@ module Datadog
         #               is very useful for testing. It may change at any point in time.
         def ip_denylist
           # Cast for Steep
-          _ = @options[:ip_denylist]
+          _ = @options[:ip_denylist] || []
+        end
+
+        # EXPERIMENTAL: This configurable is not meant to be publicly used, but
+        #               is very useful for testing. It may change at any point in time.
+        def user_id_denylist
+          # Cast for Steep
+          _ = @options[:user_id_denylist] || []
         end
 
         def waf_timeout

data/lib/datadog/appsec/configuration.rb

@@ -1,5 +1,3 @@
-# typed: ignore
-
 require_relative 'configuration/settings'
 
 module Datadog
@@ -40,6 +38,10 @@ module Datadog
           options[:ip_denylist] = value
         end
 
+        def user_id_denylist=(value)
+          options[:user_id_denylist] = value
+        end
+
         # in microseconds
         def waf_timeout=(value)
           options[:waf_timeout] = value

data/lib/datadog/appsec/contrib/auto_instrument.rb

@@ -1,5 +1,3 @@
-# typed: ignore
-
 module Datadog
   module AppSec
     module Contrib

data/lib/datadog/appsec/contrib/configuration/settings.rb

@@ -1,5 +1,3 @@
-# typed: ignore
-
 require_relative '../../../core/configuration/base'
 
 module Datadog

data/lib/datadog/appsec/contrib/integration.rb

@@ -1,5 +1,3 @@
-# typed: ignore
-
 module Datadog
   module AppSec
     module Contrib

data/lib/datadog/appsec/contrib/patcher.rb

@@ -1,5 +1,3 @@
-# typed: ignore
-
 module Datadog
   module AppSec
     module Contrib

data/lib/datadog/appsec/contrib/rack/configuration/settings.rb

@@ -1,5 +1,3 @@
-# typed: ignore
-
 require_relative '../../configuration/settings'
 require_relative '../ext'
 

data/lib/datadog/appsec/contrib/rack/ext.rb

@@ -1,5 +1,3 @@
-# typed: ignore
-
 module Datadog
   module AppSec
     module Contrib

data/lib/datadog/appsec/contrib/rack/gateway/request.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+require_relative '../../../instrumentation/gateway/argument'
+require_relative '../../../../tracing/client_ip'
+require_relative '../../../../tracing/contrib/rack/header_collection'
+
+module Datadog
+  module AppSec
+    module Contrib
+      module Rack
+        module Gateway
+          # Gateway Request argument. Normalized extration of data from Rack::Request
+          class Request < Instrumentation::Gateway::Argument
+            attr_reader :env
+
+            def initialize(env)
+              super
+              @env = env
+            end
+
+            def request
+              @request ||= ::Rack::Request.new(env)
+            end
+
+            def query
+              # Downstream libddwaf expects keys and values to be extractable
+              # separately so we can't use [[k, v], ...]. We also want to allow
+              # duplicate keys, so we use [{k, v}, ...] instead.
+              request.query_string.split('&').map do |e|
+                k, v = e.split('=').map { |s| CGI.unescape(s) }
+
+                { k => v }
+              end
+            end
+
+            # Rack < 2.0 does not have :each_header
+            # TODO: We need access to Rack here. We must make sure we are able to load AppSec without Rack,
+            # TODO: while still ensure correctness in ths code path.
+            if defined?(::Rack) && ::Rack::Request.instance_methods.include?(:each_header)
+              def headers
+                request.each_header.each_with_object({}) do |(k, v), h|
+                  h[k.gsub(/^HTTP_/, '').downcase.tr('_', '-')] = v if k =~ /^HTTP_/
+                end
+              end
+            else
+              def headers
+                request.env.each_with_object({}) do |(k, v), h|
+                  h[k.gsub(/^HTTP_/, '').downcase.tr('_', '-')] = v if k =~ /^HTTP_/
+                end
+              end
+            end
+
+            def body
+              request.body.read.tap { request.body.rewind }
+            end
+
+            def url
+              request.url
+            end
+
+            def cookies
+              request.cookies
+            end
+
+            def host
+              request.host
+            end
+
+            def user_agent
+              request.user_agent
+            end
+
+            def remote_addr
+              env['REMOTE_ADDR']
+            end
+
+            def form_hash
+              # force form data processing
+              request.POST if request.form_data?
+
+              # usually Hash<String,String> but can be a more complex
+              # Hash<String,String||Array||Hash> when e.g coming from JSON
+              env['rack.request.form_hash']
+            end
+
+            def client_ip
+              remote_ip = remote_addr
+              headers = Datadog::Tracing::Contrib::Rack::Header::RequestHeaderCollection.new(env)
+
+              result = Datadog::Tracing::ClientIp.raw_ip_from_request(headers, remote_ip)
+
+              if result.raw_ip
+                ip = Datadog::Tracing::ClientIp.strip_decorations(result.raw_ip)
+                return unless Datadog::Tracing::ClientIp.valid_ip?(ip)
+
+                ip
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/rack/gateway/response.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require_relative '../../../instrumentation/gateway/argument'
+
+module Datadog
+  module AppSec
+    module Contrib
+      module Rack
+        module Gateway
+          # Gateway Response argument.
+          class Response < Instrumentation::Gateway::Argument
+            attr_reader :body, :status, :headers, :active_context
+
+            def initialize(body, status, headers, active_context:)
+              super
+              @body = body
+              @status = status
+              @headers = headers.each_with_object({}) { |(k, v), h| h[k.downcase] = v }
+              @active_context = active_context
+            end
+
+            def response
+              @response ||= ::Rack::Response.new(body, status, headers)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/rack/gateway/watcher.rb

@@ -1,5 +1,3 @@
-# typed: ignore
-
 require_relative '../../../instrumentation/gateway'
 require_relative '../../../reactive/operation'
 require_relative '../reactive/request'
@@ -14,140 +12,144 @@ module Datadog
         module Gateway
           # Watcher for Rack gateway events
           module Watcher
-            # rubocop:disable Metrics/AbcSize
-            # rubocop:disable Metrics/MethodLength
-            # rubocop:disable Metrics/CyclomaticComplexity
-            # rubocop:disable Metrics/PerceivedComplexity
-            def self.watch
-              Instrumentation.gateway.watch('rack.request', :appsec) do |stack, request|
-                block = false
-                event = nil
-                waf_context = request.env['datadog.waf.context']
-
-                AppSec::Reactive::Operation.new('rack.request') do |op|
-                  trace = active_trace
-                  span = active_span
-
-                  Rack::Reactive::Request.subscribe(op, waf_context) do |result, _block|
-                    if result.status == :match
-                      # TODO: should this hash be an Event instance instead?
-                      event = {
-                        waf_result: result,
-                        trace: trace,
-                        span: span,
-                        request: request,
-                        actions: result.actions
-                      }
-
-                      span.set_tag('appsec.event', 'true') if span
-
-                      waf_context.events << event
+            class << self
+              def watch
+                gateway = Instrumentation.gateway
+
+                watch_request(gateway)
+                watch_response(gateway)
+                watch_request_body(gateway)
+              end
+
+              def watch_request(gateway = Instrumentation.gateway)
+                gateway.watch('rack.request', :appsec) do |stack, gateway_request|
+                  block = false
+                  event = nil
+                  waf_context = gateway_request.env['datadog.waf.context']
+
+                  AppSec::Reactive::Operation.new('rack.request') do |op|
+                    trace = active_trace
+                    span = active_span
+
+                    Rack::Reactive::Request.subscribe(op, waf_context) do |result, _block|
+                      if result.status == :match
+                        # TODO: should this hash be an Event instance instead?
+                        event = {
+                          waf_result: result,
+                          trace: trace,
+                          span: span,
+                          request: gateway_request,
+                          actions: result.actions
+                        }
+
+                        span.set_tag('appsec.event', 'true') if span
+
+                        waf_context.events << event
+                      end
                     end
+
+                    _result, block = Rack::Reactive::Request.publish(op, gateway_request)
                   end
 
-                  _result, block = Rack::Reactive::Request.publish(op, request)
-                end
+                  next [nil, [[:block, event]]] if block
 
-                next [nil, [[:block, event]]] if block
+                  ret, res = stack.call(gateway_request.request)
 
-                ret, res = stack.call(request)
+                  if event
+                    res ||= []
+                    res << [:monitor, event]
+                  end
 
-                if event
-                  res ||= []
-                  res << [:monitor, event]
+                  [ret, res]
                 end
-
-                [ret, res]
               end
 
-              Instrumentation.gateway.watch('rack.response', :appsec) do |stack, response|
-                block = false
-                event = nil
-                waf_context = response.instance_eval { @waf_context }
-
-                AppSec::Reactive::Operation.new('rack.response') do |op|
-                  trace = active_trace
-                  span = active_span
-
-                  Rack::Reactive::Response.subscribe(op, waf_context) do |result, _block|
-                    if result.status == :match
-                      # TODO: should this hash be an Event instance instead?
-                      event = {
-                        waf_result: result,
-                        trace: trace,
-                        span: span,
-                        response: response,
-                        actions: result.actions
-                      }
-
-                      span.set_tag('appsec.event', 'true') if span
-
-                      waf_context.events << event
+              def watch_response(gateway = Instrumentation.gateway)
+                gateway.watch('rack.response', :appsec) do |stack, gateway_response|
+                  block = false
+                  event = nil
+                  waf_context = gateway_response.active_context
+
+                  AppSec::Reactive::Operation.new('rack.response') do |op|
+                    trace = active_trace
+                    span = active_span
+
+                    Rack::Reactive::Response.subscribe(op, waf_context) do |result, _block|
+                      if result.status == :match
+                        # TODO: should this hash be an Event instance instead?
+                        event = {
+                          waf_result: result,
+                          trace: trace,
+                          span: span,
+                          response: gateway_response,
+                          actions: result.actions
+                        }
+
+                        span.set_tag('appsec.event', 'true') if span
+
+                        waf_context.events << event
+                      end
                     end
+
+                    _result, block = Rack::Reactive::Response.publish(op, gateway_response)
                   end
 
-                  _result, block = Rack::Reactive::Response.publish(op, response)
-                end
+                  next [nil, [[:block, event]]] if block
 
-                next [nil, [[:block, event]]] if block
+                  ret, res = stack.call(gateway_response.response)
 
-                ret, res = stack.call(response)
+                  if event
+                    res ||= []
+                    res << [:monitor, event]
+                  end
 
-                if event
-                  res ||= []
-                  res << [:monitor, event]
+                  [ret, res]
                 end
-
-                [ret, res]
               end
 
-              Instrumentation.gateway.watch('rack.request.body', :appsec) do |stack, request|
-                block = false
-                event = nil
-                waf_context = request.env['datadog.waf.context']
-
-                AppSec::Reactive::Operation.new('rack.request.body') do |op|
-                  trace = active_trace
-                  span = active_span
-
-                  Rack::Reactive::RequestBody.subscribe(op, waf_context) do |result, _block|
-                    if result.status == :match
-                      # TODO: should this hash be an Event instance instead?
-                      event = {
-                        waf_result: result,
-                        trace: trace,
-                        span: span,
-                        request: request,
-                        actions: result.actions
-                      }
-
-                      span.set_tag('appsec.event', 'true') if span
-
-                      waf_context.events << event
+              def watch_request_body(gateway = Instrumentation.gateway)
+                gateway.watch('rack.request.body', :appsec) do |stack, gateway_request|
+                  block = false
+                  event = nil
+                  waf_context = gateway_request.env['datadog.waf.context']
+
+                  AppSec::Reactive::Operation.new('rack.request.body') do |op|
+                    trace = active_trace
+                    span = active_span
+
+                    Rack::Reactive::RequestBody.subscribe(op, waf_context) do |result, _block|
+                      if result.status == :match
+                        # TODO: should this hash be an Event instance instead?
+                        event = {
+                          waf_result: result,
+                          trace: trace,
+                          span: span,
+                          request: gateway_request,
+                          actions: result.actions
+                        }
+
+                        span.set_tag('appsec.event', 'true') if span
+
+                        waf_context.events << event
+                      end
                     end
+
+                    _result, block = Rack::Reactive::RequestBody.publish(op, gateway_request)
                   end
 
-                  _result, block = Rack::Reactive::RequestBody.publish(op, request)
-                end
+                  next [nil, [[:block, event]]] if block
 
-                next [nil, [[:block, event]]] if block
+                  ret, res = stack.call(gateway_request.request)
 
-                ret, res = stack.call(request)
+                  if event
+                    res ||= []
+                    res << [:monitor, event]
+                  end
 
-                if event
-                  res ||= []
-                  res << [:monitor, event]
+                  [ret, res]
                 end
-
-                [ret, res]
               end
-            end
-            # rubocop:enable Metrics/CyclomaticComplexity
-            # rubocop:enable Metrics/PerceivedComplexity
-            # rubocop:enable Metrics/MethodLength
-            # rubocop:enable Metrics/AbcSize
 
-            class << self
               private
 
               def active_trace

data/lib/datadog/appsec/contrib/rack/integration.rb

@@ -1,5 +1,3 @@
-# typed: ignore
-
 require_relative '../integration'
 
 require_relative 'configuration/settings'

data/lib/datadog/appsec/contrib/rack/patcher.rb

@@ -1,6 +1,5 @@
-# typed: ignore
-
 require_relative '../patcher'
+require_relative '../../monitor'
 require_relative 'gateway/watcher'
 
 module Datadog
@@ -22,6 +21,7 @@ module Datadog
           end
 
           def patch
+            Monitor::Gateway::Watcher.watch
             Gateway::Watcher.watch
             Patcher.instance_variable_set(:@patched, true)
           end