ddtrace 1.6.1 → 1.7.0

data/lib/datadog/appsec/assets/blocked.json

@@ -0,0 +1 @@
+{"errors": [{"title": "You've been blocked", "detail": "Sorry, you cannot access this page. Please contact the customer service team. Security provided by Datadog."}]}

data/lib/datadog/appsec/assets/blocked.text

@@ -0,0 +1,5 @@
+You've been blocked
+
+Sorry, you cannot access this page. Please contact the customer service team.
+
+Security provided by Datadog.

data/lib/datadog/appsec/assets/waf_rules/recommended.json

@@ -1,7 +1,7 @@
 {
   "version": "2.2",
   "metadata": {
-    "rules_version": "1.4.1"
+    "rules_version": "1.4.2"
   },
   "rules": [
     {
@@ -2853,51 +2853,6 @@
       ],
       "transformers": []
     },
-    {
-      "id": "crs-941-100",
-      "name": "XSS Attack Detected via libinjection",
-      "tags": {
-        "type": "xss",
-        "crs_id": "941100",
-        "category": "attack_attempt"
-      },
-      "conditions": [
-        {
-          "parameters": {
-            "inputs": [
-              {
-                "address": "server.request.headers.no_cookies",
-                "key_path": [
-                  "user-agent"
-                ]
-              },
-              {
-                "address": "server.request.headers.no_cookies",
-                "key_path": [
-                  "referer"
-                ]
-              },
-              {
-                "address": "server.request.query"
-              },
-              {
-                "address": "server.request.body"
-              },
-              {
-                "address": "server.request.path_params"
-              },
-              {
-                "address": "grpc.server.request.message"
-              }
-            ]
-          },
-          "operator": "is_xss"
-        }
-      ],
-      "transformers": [
-        "removeNulls"
-      ]
-    },
     {
       "id": "crs-941-110",
       "name": "XSS Filter - Category 1: Script Tag Vector",
@@ -4363,6 +4318,40 @@
         "keys_only"
       ]
     },
+    {
+      "id": "dog-000-007",
+      "name": "Server side template injection: Velocity & Freemarker",
+      "tags": {
+        "type": "java_code_injection",
+        "category": "attack_attempt"
+      },
+      "conditions": [
+        {
+          "parameters": {
+            "inputs": [
+              {
+                "address": "server.request.query"
+              },
+              {
+                "address": "server.request.body"
+              },
+              {
+                "address": "server.request.path_params"
+              },
+              {
+                "address": "server.request.headers.no_cookies"
+              },
+              {
+                "address": "grpc.server.request.message"
+              }
+            ],
+            "regex": "#(?:set|foreach|macro|parse|if)\\(.*\\)|<#assign.*>"
+          },
+          "operator": "match_regex"
+        }
+      ],
+      "transformers": []
+    },
     {
       "id": "nfd-000-001",
       "name": "Detect common directory discovery scans",

data/lib/datadog/appsec/assets/waf_rules/risky.json

@@ -1,7 +1,7 @@
 {
   "version": "2.2",
   "metadata": {
-    "rules_version": "1.4.1"
+    "rules_version": "1.4.2"
   },
   "rules": [
     {

data/lib/datadog/appsec/assets/waf_rules/strict.json

@@ -1,7 +1,7 @@
 {
   "version": "2.2",
   "metadata": {
-    "rules_version": "1.4.1"
+    "rules_version": "1.4.2"
   },
   "rules": [
     {
@@ -855,6 +855,51 @@
       ],
       "transformers": []
     },
+    {
+      "id": "crs-941-100",
+      "name": "XSS Attack Detected via libinjection",
+      "tags": {
+        "type": "xss",
+        "crs_id": "941100",
+        "category": "attack_attempt"
+      },
+      "conditions": [
+        {
+          "parameters": {
+            "inputs": [
+              {
+                "address": "server.request.headers.no_cookies",
+                "key_path": [
+                  "user-agent"
+                ]
+              },
+              {
+                "address": "server.request.headers.no_cookies",
+                "key_path": [
+                  "referer"
+                ]
+              },
+              {
+                "address": "server.request.query"
+              },
+              {
+                "address": "server.request.body"
+              },
+              {
+                "address": "server.request.path_params"
+              },
+              {
+                "address": "grpc.server.request.message"
+              }
+            ]
+          },
+          "operator": "is_xss"
+        }
+      ],
+      "transformers": [
+        "removeNulls"
+      ]
+    },
     {
       "id": "crs-941-130",
       "name": "XSS Filter - Category 3: Attribute Vector",

data/lib/datadog/appsec/assets.rb

@@ -12,8 +12,8 @@ module Datadog
         read("waf_rules/#{kind}.json")
       end
 
-      def blocked
-        @blocked ||= read('blocked.html')
+      def blocked(format: :html)
+        (@blocked ||= {})[format] ||= read("blocked.#{format}")
       end
 
       def path

data/lib/datadog/appsec/configuration/settings.rb

@@ -128,6 +128,12 @@ module Datadog
           @options[:ruleset]
         end
 
+        # EXPERIMENTAL: This configurable is not meant to be publicly used, but
+        #               is very useful for testing. It may change at any point in time.
+        def ip_denylist
+          @options[:ip_denylist]
+        end
+
         def waf_timeout
           @options[:waf_timeout]
         end

data/lib/datadog/appsec/configuration.rb

@@ -34,6 +34,10 @@ module Datadog
           options[:ruleset] = value
         end
 
+        def ip_denylist=(value)
+          options[:ip_denylist] = value
+        end
+
         # in microseconds
         def waf_timeout=(value)
           options[:waf_timeout] = value

data/lib/datadog/appsec/contrib/rack/reactive/request.rb

@@ -15,9 +15,7 @@ module Datadog
                 op.publish('request.headers', Rack::Request.headers(request))
                 op.publish('request.uri.raw', Rack::Request.url(request))
                 op.publish('request.cookies', Rack::Request.cookies(request))
-                # op.publish('request.body.raw', Rack::Request.body(request))
-                # TODO: op.publish('request.path_params', { k: v }) # route params only?
-                # TODO: op.publish('request.path', request.script_name + request.path) # unused for now
+                op.publish('request.client_ip', Rack::Request.client_ip(request))
 
                 nil
               end
@@ -30,8 +28,7 @@ module Datadog
                 'request.uri.raw',
                 'request.query',
                 'request.cookies',
-                # 'request.body.raw',
-                # TODO: 'request.path_params',
+                'request.client_ip',
               ]
 
               op.subscribe(*addresses) do |*values|
@@ -41,16 +38,15 @@ module Datadog
                 uri_raw = values[1]
                 query = values[2]
                 cookies = values[3]
-                # body = values[4]
+                client_ip = values[4]
 
                 waf_args = {
                   'server.request.cookies' => cookies,
-                  # 'server.request.body.raw' => body,
                   'server.request.query' => query,
                   'server.request.uri.raw' => uri_raw,
                   'server.request.headers' => headers,
                   'server.request.headers.no_cookies' => headers_no_cookies,
-                  # TODO: 'server.request.path_params' => path_params,
+                  'http.client_ip' => client_ip,
                 }
 
                 waf_timeout = Datadog::AppSec.settings.waf_timeout

data/lib/datadog/appsec/contrib/rack/request.rb

@@ -1,5 +1,8 @@
 # typed: true
 
+require_relative '../../../tracing/client_ip'
+require_relative '../../../tracing/contrib/rack/header_collection'
+
 module Datadog
   module AppSec
     module Contrib
@@ -54,6 +57,20 @@ module Datadog
             # Hash<String,String||Array||Hash> when e.g coming from JSON
             request.env['rack.request.form_hash']
           end
+
+          def self.client_ip(request)
+            remote_ip = request.env['REMOTE_ADDR']
+            headers = Datadog::Tracing::Contrib::Rack::Header::RequestHeaderCollection.new(request.env)
+
+            result = Datadog::Tracing::ClientIp.raw_ip_from_request(headers, remote_ip)
+
+            if result.raw_ip
+              ip = Datadog::Tracing::ClientIp.strip_decorations(result.raw_ip)
+              return unless Datadog::Tracing::ClientIp.valid_ip?(ip)
+
+              ip
+            end
+          end
         end
       end
     end

data/lib/datadog/appsec/contrib/rack/request_body_middleware.rb

@@ -1,7 +1,7 @@
 # typed: ignore
 
 require_relative '../../instrumentation/gateway'
-require_relative '../../assets'
+require_relative '../../response'
 
 module Datadog
   module AppSec
@@ -29,7 +29,7 @@ module Datadog
             end
 
             if request_response && request_response.any? { |action, _event| action == :block }
-              request_return = [403, { 'Content-Type' => 'text/html' }, [Datadog::AppSec::Assets.blocked]]
+              request_return = AppSec::Response.negotiate(env).to_rack
             end
 
             request_return

data/lib/datadog/appsec/contrib/rack/request_middleware.rb

@@ -4,7 +4,7 @@ require 'json'
 
 require_relative '../../instrumentation/gateway'
 require_relative '../../processor'
-require_relative '../../assets'
+require_relative '../../response'
 
 require_relative '../../../tracing/client_ip'
 require_relative '../../../tracing/contrib/rack/header_collection'
@@ -40,7 +40,7 @@ module Datadog
             end
 
             if request_response && request_response.any? { |action, _event| action == :block }
-              request_return = [403, { 'Content-Type' => 'text/html' }, [Datadog::AppSec::Assets.blocked]]
+              request_return = AppSec::Response.negotiate(env).to_rack
             end
 
             response = ::Rack::Response.new(request_return[2], request_return[0], request_return[1])

data/lib/datadog/appsec/contrib/rails/patcher.rb

@@ -4,6 +4,7 @@ require_relative '../../../core/utils/only_once'
 
 require_relative '../patcher'
 require_relative 'framework'
+require_relative '../../response'
 require_relative '../rack/request_middleware'
 require_relative '../rack/request_body_middleware'
 require_relative 'gateway/watcher'
@@ -83,11 +84,7 @@ module Datadog
               end
 
               if request_response && request_response.any? { |action, _event| action == :block }
-                @_response = ::ActionDispatch::Response.new(
-                  403,
-                  { 'Content-Type' => 'text/html' },
-                  [Datadog::AppSec::Assets.blocked]
-                )
+                @_response = AppSec::Response.negotiate(env).to_action_dispatch_response
                 request_return = @_response.body
               end
 
@@ -96,7 +93,7 @@ module Datadog
           end
 
           def patch_process_action
-            ActionController::Instrumentation.prepend(ProcessActionPatch)
+            ::ActionController::Metal.prepend(ProcessActionPatch)
           end
 
           def include_middleware?(middleware, app)

data/lib/datadog/appsec/contrib/sinatra/ext.rb

@@ -8,6 +8,7 @@ module Datadog
         module Ext
           APP = 'sinatra'.freeze
           ENV_ENABLED = 'DD_TRACE_SINATRA_ENABLED'.freeze
+          ROUTE_INTERRUPT = :datadog_appsec_contrib_sinatra_route_interrupt
         end
       end
     end

data/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb

@@ -11,7 +11,7 @@ module Datadog
     module Contrib
       module Sinatra
         module Gateway
-          # Watcher for Rails gateway events
+          # Watcher for Sinatra gateway events
           module Watcher
             # rubocop:disable Metrics/MethodLength
             def self.watch

data/lib/datadog/appsec/contrib/sinatra/patcher.rb

@@ -3,6 +3,7 @@
 require_relative '../../../tracing/contrib/rack/middlewares'
 
 require_relative '../patcher'
+require_relative '../../response'
 require_relative '../rack/request_middleware'
 require_relative 'framework'
 require_relative 'gateway/watcher'
@@ -57,15 +58,12 @@ module Datadog
             # TODO: handle exceptions, except for super
 
             request_return, request_response = Instrumentation.gateway.push('sinatra.request.dispatch', request) do
-              super
+              # handle process_route interruption
+              catch(Ext::ROUTE_INTERRUPT) { super }
             end
 
             if request_response && request_response.any? { |action, _event| action == :block }
-              self.response = ::Sinatra::Response.new(
-                [Datadog::AppSec::Assets.blocked],
-                403,
-                { 'Content-Type' => 'text/html' }
-              )
+              self.response = AppSec::Response.negotiate(env).to_sinatra_response
               request_return = nil
             end
 
@@ -94,9 +92,14 @@ module Datadog
               # At this point params has both route params and normal params.
               route_params = params.each.with_object({}) { |(k, v), h| h[k] = v unless base_params.key?(k) }
 
-              Instrumentation.gateway.push('sinatra.request.routed', [request, route_params])
+              _, request_response = Instrumentation.gateway.push('sinatra.request.routed', [request, route_params])
 
-              # TODO: handle block
+              if request_response && request_response.any? { |action, _event| action == :block }
+                self.response = AppSec::Response.negotiate(env).to_sinatra_response
+
+                # interrupt request and return response to dispatch! for consistency
+                throw(Ext::ROUTE_INTERRUPT, response)
+              end
 
               yield(*args)
             end

data/lib/datadog/appsec/extensions.rb

@@ -53,6 +53,12 @@ module Datadog
           @settings.merge(dsl)
         end
 
+        def ip_denylist=(arg)
+          dsl = AppSec::Configuration::DSL.new
+          dsl.ip_denylist = arg
+          @settings.merge(dsl)
+        end
+
         def instrument(*args)
           dsl = AppSec::Configuration::DSL.new
           dsl.instrument(*args)
@@ -86,6 +92,10 @@ module Datadog
           @settings.ruleset
         end
 
+        def ruledata
+          @settings.ruledata
+        end
+
         def waf_timeout
           @settings.waf_timeout
         end

data/lib/datadog/appsec/processor.rb

@@ -57,7 +57,11 @@ module Datadog
 
         unless load_libddwaf && load_ruleset && create_waf_handle
           Datadog.logger.warn { 'AppSec is disabled, see logged errors above' }
+
+          return
         end
+
+        update_ip_denylist
       end
 
       def ready?
@@ -76,6 +80,20 @@ module Datadog
         @handle.toggle_rules(map)
       end
 
+      def update_ip_denylist(denylist = Datadog::AppSec.settings.ip_denylist, id: 'blocked_ips')
+        denylist ||= []
+
+        ruledata_setting = [
+          {
+            'id' => id,
+            'type' => 'data_with_expiration',
+            'data' => denylist.map { |ip| { 'value' => ip.to_s, 'expiration' => 2**63 } }
+          }
+        ]
+
+        update_rule_data(ruledata_setting)
+      end
+
       def finalize
         @handle.finalize
       end

data/lib/datadog/appsec/response.rb

@@ -0,0 +1,54 @@
+# typed: false
+
+require_relative 'assets'
+
+module Datadog
+  module AppSec
+    # AppSec response
+    class Response
+      attr_reader :status, :headers, :body
+
+      def initialize(status:, headers: {}, body: [])
+        @status = status
+        @headers = headers
+        @body = body
+      end
+
+      def to_rack
+        [status, headers, body]
+      end
+
+      def to_sinatra_response
+        ::Sinatra::Response.new(body, status, headers)
+      end
+
+      def to_action_dispatch_response
+        ::ActionDispatch::Response.new(status, headers, body)
+      end
+
+      class << self
+        def negotiate(env)
+          Response.new(
+            status: 403,
+            headers: { 'Content-Type' => 'text/html' },
+            body: [Datadog::AppSec::Assets.blocked(format: format(env))]
+          )
+        end
+
+        private
+
+        def format(env)
+          format = env['HTTP_ACCEPT'] && env['HTTP_ACCEPT'].split(',').any? do |accept|
+            if accept.start_with?('text/html')
+              break :html
+            elsif accept.start_with?('application/json')
+              break :json
+            end
+          end
+
+          format || :text
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/runtime/ext.rb

@@ -7,7 +7,7 @@ module Datadog
       module Ext
         TAG_ID = 'runtime-id'.freeze
         TAG_LANG = 'language'.freeze
-        TAG_PID = 'system.pid'.freeze
+        TAG_PROCESS_ID = 'process_id'.freeze
 
         # Metrics
         # @public_api

data/lib/datadog/opentracer/distributed_headers.rb

@@ -1,15 +1,13 @@
 # typed: true
 
 require_relative '../tracing/span'
-require_relative '../tracing/distributed/headers/ext'
+require_relative '../tracing/distributed/datadog'
 
 module Datadog
   module OpenTracer
     # DistributedHeaders provides easy access and validation to headers
     # @public_api
     class DistributedHeaders
-      include Tracing::Distributed::Headers::Ext
-
       def initialize(carrier)
         @carrier = carrier
       end
@@ -20,15 +18,15 @@ module Datadog
       end
 
       def trace_id
-        id HTTP_HEADER_TRACE_ID
+        id Tracing::Distributed::Datadog::TRACE_ID_KEY
       end
 
       def parent_id
-        id HTTP_HEADER_PARENT_ID
+        id Tracing::Distributed::Datadog::PARENT_ID_KEY
       end
 
       def sampling_priority
-        hdr = @carrier[HTTP_HEADER_SAMPLING_PRIORITY]
+        hdr = @carrier[Tracing::Distributed::Datadog::SAMPLING_PRIORITY_KEY]
         # It's important to make a difference between no header,
         # and a header defined to zero.
         return unless hdr
@@ -40,7 +38,7 @@ module Datadog
       end
 
       def origin
-        hdr = @carrier[HTTP_HEADER_ORIGIN]
+        hdr = @carrier[Tracing::Distributed::Datadog::ORIGIN_KEY]
         # Only return the value if it is not an empty string
         hdr if hdr != ''
       end

data/lib/datadog/opentracer/rack_propagator.rb

@@ -1,7 +1,6 @@
 # typed: true
 
 require_relative '../tracing/context'
-require_relative '../tracing/distributed/headers/ext'
 require_relative '../tracing/propagation/http'
 require_relative '../tracing/trace_operation'
 require_relative 'propagator'
@@ -11,8 +10,6 @@ module Datadog
     # OpenTracing propagator for Datadog::OpenTracer::Tracer
     module RackPropagator
       extend Propagator
-      extend Tracing::Distributed::Headers::Ext
-      include Tracing::Distributed::Headers::Ext
 
       BAGGAGE_PREFIX = 'ot-baggage-'.freeze
       BAGGAGE_PREFIX_FORMATTED = 'HTTP_OT_BAGGAGE_'.freeze

data/lib/datadog/opentracer/text_map_propagator.rb

@@ -1,7 +1,7 @@
 # typed: true
 
 require_relative '../tracing/context'
-require_relative '../tracing/distributed/headers/ext'
+require_relative '../tracing/distributed/datadog'
 require_relative '../tracing/trace_operation'
 require_relative 'propagator'
 
@@ -10,8 +10,6 @@ module Datadog
     # OpenTracing propagator for Datadog::OpenTracer::Tracer
     module TextMapPropagator
       extend Propagator
-      extend Tracing::Distributed::Headers::Ext
-      include Tracing::Distributed::Headers::Ext
 
       BAGGAGE_PREFIX = 'ot-baggage-'.freeze
 
@@ -34,10 +32,10 @@ module Datadog
                    end
           return unless digest
 
-          carrier[HTTP_HEADER_ORIGIN] = digest.trace_origin
-          carrier[HTTP_HEADER_PARENT_ID] = digest.span_id
-          carrier[HTTP_HEADER_SAMPLING_PRIORITY] = digest.trace_sampling_priority
-          carrier[HTTP_HEADER_TRACE_ID] = digest.trace_id
+          carrier[Tracing::Distributed::Datadog::ORIGIN_KEY] = digest.trace_origin
+          carrier[Tracing::Distributed::Datadog::PARENT_ID_KEY] = digest.span_id
+          carrier[Tracing::Distributed::Datadog::SAMPLING_PRIORITY_KEY] = digest.trace_sampling_priority
+          carrier[Tracing::Distributed::Datadog::TRACE_ID_KEY] = digest.trace_id
 
           nil
         end

data/lib/datadog/profiling/collectors/cpu_and_wall_time.rb

@@ -23,13 +23,19 @@ module Datadog
           result
         end
 
+        def reset_after_fork
+          self.class._native_reset_after_fork(self)
+        end
+
         private
 
         def safely_extract_context_key_from(tracer)
-          tracer &&
-            tracer.respond_to?(:provider) &&
-            # NOTE: instance_variable_get always works, even on nil -- it just returns nil if the variable doesn't exist
-            tracer.provider.instance_variable_get(:@context).instance_variable_get(:@key)
+          provider = tracer && tracer.respond_to?(:provider) && tracer.provider
+
+          return unless provider
+
+          context = provider.instance_variable_get(:@context)
+          context && context.instance_variable_get(:@key)
         end
       end
     end

data/lib/datadog/profiling/collectors/cpu_and_wall_time_worker.rb

@@ -70,6 +70,10 @@ module Datadog
             @failure_exception = nil
           end
         end
+
+        def reset_after_fork
+          self.class._native_reset_after_fork(self)
+        end
       end
     end
   end