ddtrace 1.6.1 → 1.8.0

data/lib/datadog/appsec/assets/blocked.json

@@ -0,0 +1 @@
+{"errors": [{"title": "You've been blocked", "detail": "Sorry, you cannot access this page. Please contact the customer service team. Security provided by Datadog."}]}

data/lib/datadog/appsec/assets/blocked.text

@@ -0,0 +1,5 @@
+You've been blocked
+
+Sorry, you cannot access this page. Please contact the customer service team.
+
+Security provided by Datadog.

data/lib/datadog/appsec/assets/waf_rules/recommended.json

@@ -1,7 +1,7 @@
 {
   "version": "2.2",
   "metadata": {
-    "rules_version": "1.4.1"
+    "rules_version": "1.4.2"
   },
   "rules": [
     {
@@ -2853,51 +2853,6 @@
       ],
       "transformers": []
     },
-    {
-      "id": "crs-941-100",
-      "name": "XSS Attack Detected via libinjection",
-      "tags": {
-        "type": "xss",
-        "crs_id": "941100",
-        "category": "attack_attempt"
-      },
-      "conditions": [
-        {
-          "parameters": {
-            "inputs": [
-              {
-                "address": "server.request.headers.no_cookies",
-                "key_path": [
-                  "user-agent"
-                ]
-              },
-              {
-                "address": "server.request.headers.no_cookies",
-                "key_path": [
-                  "referer"
-                ]
-              },
-              {
-                "address": "server.request.query"
-              },
-              {
-                "address": "server.request.body"
-              },
-              {
-                "address": "server.request.path_params"
-              },
-              {
-                "address": "grpc.server.request.message"
-              }
-            ]
-          },
-          "operator": "is_xss"
-        }
-      ],
-      "transformers": [
-        "removeNulls"
-      ]
-    },
     {
       "id": "crs-941-110",
       "name": "XSS Filter - Category 1: Script Tag Vector",
@@ -4363,6 +4318,40 @@
         "keys_only"
       ]
     },
+    {
+      "id": "dog-000-007",
+      "name": "Server side template injection: Velocity & Freemarker",
+      "tags": {
+        "type": "java_code_injection",
+        "category": "attack_attempt"
+      },
+      "conditions": [
+        {
+          "parameters": {
+            "inputs": [
+              {
+                "address": "server.request.query"
+              },
+              {
+                "address": "server.request.body"
+              },
+              {
+                "address": "server.request.path_params"
+              },
+              {
+                "address": "server.request.headers.no_cookies"
+              },
+              {
+                "address": "grpc.server.request.message"
+              }
+            ],
+            "regex": "#(?:set|foreach|macro|parse|if)\\(.*\\)|<#assign.*>"
+          },
+          "operator": "match_regex"
+        }
+      ],
+      "transformers": []
+    },
     {
       "id": "nfd-000-001",
       "name": "Detect common directory discovery scans",

data/lib/datadog/appsec/assets/waf_rules/risky.json

@@ -1,7 +1,7 @@
 {
   "version": "2.2",
   "metadata": {
-    "rules_version": "1.4.1"
+    "rules_version": "1.4.2"
   },
   "rules": [
     {

data/lib/datadog/appsec/assets/waf_rules/strict.json

@@ -1,7 +1,7 @@
 {
   "version": "2.2",
   "metadata": {
-    "rules_version": "1.4.1"
+    "rules_version": "1.4.2"
   },
   "rules": [
     {
@@ -855,6 +855,51 @@
       ],
       "transformers": []
     },
+    {
+      "id": "crs-941-100",
+      "name": "XSS Attack Detected via libinjection",
+      "tags": {
+        "type": "xss",
+        "crs_id": "941100",
+        "category": "attack_attempt"
+      },
+      "conditions": [
+        {
+          "parameters": {
+            "inputs": [
+              {
+                "address": "server.request.headers.no_cookies",
+                "key_path": [
+                  "user-agent"
+                ]
+              },
+              {
+                "address": "server.request.headers.no_cookies",
+                "key_path": [
+                  "referer"
+                ]
+              },
+              {
+                "address": "server.request.query"
+              },
+              {
+                "address": "server.request.body"
+              },
+              {
+                "address": "server.request.path_params"
+              },
+              {
+                "address": "grpc.server.request.message"
+              }
+            ]
+          },
+          "operator": "is_xss"
+        }
+      ],
+      "transformers": [
+        "removeNulls"
+      ]
+    },
     {
       "id": "crs-941-130",
       "name": "XSS Filter - Category 3: Attribute Vector",

data/lib/datadog/appsec/assets.rb

@@ -12,8 +12,8 @@ module Datadog
         read("waf_rules/#{kind}.json")
       end
 
-      def blocked
-        @blocked ||= read('blocked.html')
+      def blocked(format: :html)
+        (@blocked ||= {})[format] ||= read("blocked.#{format}")
       end
 
       def path

data/lib/datadog/appsec/configuration/settings.rb

@@ -128,6 +128,12 @@ module Datadog
           @options[:ruleset]
         end
 
+        # EXPERIMENTAL: This configurable is not meant to be publicly used, but
+        #               is very useful for testing. It may change at any point in time.
+        def ip_denylist
+          @options[:ip_denylist]
+        end
+
         def waf_timeout
           @options[:waf_timeout]
         end

data/lib/datadog/appsec/configuration.rb

@@ -34,6 +34,10 @@ module Datadog
           options[:ruleset] = value
         end
 
+        def ip_denylist=(value)
+          options[:ip_denylist] = value
+        end
+
         # in microseconds
         def waf_timeout=(value)
           options[:waf_timeout] = value

data/lib/datadog/appsec/contrib/rack/reactive/request.rb

@@ -15,9 +15,7 @@ module Datadog
                 op.publish('request.headers', Rack::Request.headers(request))
                 op.publish('request.uri.raw', Rack::Request.url(request))
                 op.publish('request.cookies', Rack::Request.cookies(request))
-                # op.publish('request.body.raw', Rack::Request.body(request))
-                # TODO: op.publish('request.path_params', { k: v }) # route params only?
-                # TODO: op.publish('request.path', request.script_name + request.path) # unused for now
+                op.publish('request.client_ip', Rack::Request.client_ip(request))
 
                 nil
               end
@@ -30,8 +28,7 @@ module Datadog
                 'request.uri.raw',
                 'request.query',
                 'request.cookies',
-                # 'request.body.raw',
-                # TODO: 'request.path_params',
+                'request.client_ip',
               ]
 
               op.subscribe(*addresses) do |*values|
@@ -41,16 +38,15 @@ module Datadog
                 uri_raw = values[1]
                 query = values[2]
                 cookies = values[3]
-                # body = values[4]
+                client_ip = values[4]
 
                 waf_args = {
                   'server.request.cookies' => cookies,
-                  # 'server.request.body.raw' => body,
                   'server.request.query' => query,
                   'server.request.uri.raw' => uri_raw,
                   'server.request.headers' => headers,
                   'server.request.headers.no_cookies' => headers_no_cookies,
-                  # TODO: 'server.request.path_params' => path_params,
+                  'http.client_ip' => client_ip,
                 }
 
                 waf_timeout = Datadog::AppSec.settings.waf_timeout

data/lib/datadog/appsec/contrib/rack/request.rb

@@ -1,5 +1,8 @@
 # typed: true
 
+require_relative '../../../tracing/client_ip'
+require_relative '../../../tracing/contrib/rack/header_collection'
+
 module Datadog
   module AppSec
     module Contrib
@@ -54,6 +57,20 @@ module Datadog
             # Hash<String,String||Array||Hash> when e.g coming from JSON
             request.env['rack.request.form_hash']
           end
+
+          def self.client_ip(request)
+            remote_ip = request.env['REMOTE_ADDR']
+            headers = Datadog::Tracing::Contrib::Rack::Header::RequestHeaderCollection.new(request.env)
+
+            result = Datadog::Tracing::ClientIp.raw_ip_from_request(headers, remote_ip)
+
+            if result.raw_ip
+              ip = Datadog::Tracing::ClientIp.strip_decorations(result.raw_ip)
+              return unless Datadog::Tracing::ClientIp.valid_ip?(ip)
+
+              ip
+            end
+          end
         end
       end
     end

data/lib/datadog/appsec/contrib/rack/request_body_middleware.rb

@@ -1,7 +1,7 @@
 # typed: ignore
 
 require_relative '../../instrumentation/gateway'
-require_relative '../../assets'
+require_relative '../../response'
 
 module Datadog
   module AppSec
@@ -29,7 +29,7 @@ module Datadog
             end
 
             if request_response && request_response.any? { |action, _event| action == :block }
-              request_return = [403, { 'Content-Type' => 'text/html' }, [Datadog::AppSec::Assets.blocked]]
+              request_return = AppSec::Response.negotiate(env).to_rack
             end
 
             request_return

data/lib/datadog/appsec/contrib/rack/request_middleware.rb

@@ -4,7 +4,7 @@ require 'json'
 
 require_relative '../../instrumentation/gateway'
 require_relative '../../processor'
-require_relative '../../assets'
+require_relative '../../response'
 
 require_relative '../../../tracing/client_ip'
 require_relative '../../../tracing/contrib/rack/header_collection'
@@ -40,7 +40,7 @@ module Datadog
             end
 
             if request_response && request_response.any? { |action, _event| action == :block }
-              request_return = [403, { 'Content-Type' => 'text/html' }, [Datadog::AppSec::Assets.blocked]]
+              request_return = AppSec::Response.negotiate(env).to_rack
             end
 
             response = ::Rack::Response.new(request_return[2], request_return[0], request_return[1])

data/lib/datadog/appsec/contrib/rails/patcher.rb

@@ -4,6 +4,7 @@ require_relative '../../../core/utils/only_once'
 
 require_relative '../patcher'
 require_relative 'framework'
+require_relative '../../response'
 require_relative '../rack/request_middleware'
 require_relative '../rack/request_body_middleware'
 require_relative 'gateway/watcher'
@@ -83,11 +84,7 @@ module Datadog
               end
 
               if request_response && request_response.any? { |action, _event| action == :block }
-                @_response = ::ActionDispatch::Response.new(
-                  403,
-                  { 'Content-Type' => 'text/html' },
-                  [Datadog::AppSec::Assets.blocked]
-                )
+                @_response = AppSec::Response.negotiate(env).to_action_dispatch_response
                 request_return = @_response.body
               end
 
@@ -96,7 +93,7 @@ module Datadog
           end
 
           def patch_process_action
-            ActionController::Instrumentation.prepend(ProcessActionPatch)
+            ::ActionController::Metal.prepend(ProcessActionPatch)
           end
 
           def include_middleware?(middleware, app)

data/lib/datadog/appsec/contrib/sinatra/ext.rb

@@ -8,6 +8,7 @@ module Datadog
         module Ext
           APP = 'sinatra'.freeze
           ENV_ENABLED = 'DD_TRACE_SINATRA_ENABLED'.freeze
+          ROUTE_INTERRUPT = :datadog_appsec_contrib_sinatra_route_interrupt
         end
       end
     end

data/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb

@@ -11,7 +11,7 @@ module Datadog
     module Contrib
       module Sinatra
         module Gateway
-          # Watcher for Rails gateway events
+          # Watcher for Sinatra gateway events
           module Watcher
             # rubocop:disable Metrics/MethodLength
             def self.watch

data/lib/datadog/appsec/contrib/sinatra/patcher.rb

@@ -3,6 +3,7 @@
 require_relative '../../../tracing/contrib/rack/middlewares'
 
 require_relative '../patcher'
+require_relative '../../response'
 require_relative '../rack/request_middleware'
 require_relative 'framework'
 require_relative 'gateway/watcher'
@@ -57,15 +58,12 @@ module Datadog
             # TODO: handle exceptions, except for super
 
             request_return, request_response = Instrumentation.gateway.push('sinatra.request.dispatch', request) do
-              super
+              # handle process_route interruption
+              catch(Ext::ROUTE_INTERRUPT) { super }
             end
 
             if request_response && request_response.any? { |action, _event| action == :block }
-              self.response = ::Sinatra::Response.new(
-                [Datadog::AppSec::Assets.blocked],
-                403,
-                { 'Content-Type' => 'text/html' }
-              )
+              self.response = AppSec::Response.negotiate(env).to_sinatra_response
               request_return = nil
             end
 
@@ -94,9 +92,14 @@ module Datadog
               # At this point params has both route params and normal params.
               route_params = params.each.with_object({}) { |(k, v), h| h[k] = v unless base_params.key?(k) }
 
-              Instrumentation.gateway.push('sinatra.request.routed', [request, route_params])
+              _, request_response = Instrumentation.gateway.push('sinatra.request.routed', [request, route_params])
 
-              # TODO: handle block
+              if request_response && request_response.any? { |action, _event| action == :block }
+                self.response = AppSec::Response.negotiate(env).to_sinatra_response
+
+                # interrupt request and return response to dispatch! for consistency
+                throw(Ext::ROUTE_INTERRUPT, response)
+              end
 
               yield(*args)
             end

data/lib/datadog/appsec/extensions.rb

@@ -53,6 +53,12 @@ module Datadog
           @settings.merge(dsl)
         end
 
+        def ip_denylist=(arg)
+          dsl = AppSec::Configuration::DSL.new
+          dsl.ip_denylist = arg
+          @settings.merge(dsl)
+        end
+
         def instrument(*args)
           dsl = AppSec::Configuration::DSL.new
           dsl.instrument(*args)
@@ -86,6 +92,10 @@ module Datadog
           @settings.ruleset
         end
 
+        def ruledata
+          @settings.ruledata
+        end
+
         def waf_timeout
           @settings.waf_timeout
         end

data/lib/datadog/appsec/processor.rb

@@ -57,7 +57,11 @@ module Datadog
 
         unless load_libddwaf && load_ruleset && create_waf_handle
           Datadog.logger.warn { 'AppSec is disabled, see logged errors above' }
+
+          return
         end
+
+        update_ip_denylist
       end
 
       def ready?
@@ -76,6 +80,20 @@ module Datadog
         @handle.toggle_rules(map)
       end
 
+      def update_ip_denylist(denylist = Datadog::AppSec.settings.ip_denylist, id: 'blocked_ips')
+        denylist ||= []
+
+        ruledata_setting = [
+          {
+            'id' => id,
+            'type' => 'data_with_expiration',
+            'data' => denylist.map { |ip| { 'value' => ip.to_s, 'expiration' => 2**63 } }
+          }
+        ]
+
+        update_rule_data(ruledata_setting)
+      end
+
       def finalize
         @handle.finalize
       end

data/lib/datadog/appsec/response.rb

@@ -0,0 +1,54 @@
+# typed: false
+
+require_relative 'assets'
+
+module Datadog
+  module AppSec
+    # AppSec response
+    class Response
+      attr_reader :status, :headers, :body
+
+      def initialize(status:, headers: {}, body: [])
+        @status = status
+        @headers = headers
+        @body = body
+      end
+
+      def to_rack
+        [status, headers, body]
+      end
+
+      def to_sinatra_response
+        ::Sinatra::Response.new(body, status, headers)
+      end
+
+      def to_action_dispatch_response
+        ::ActionDispatch::Response.new(status, headers, body)
+      end
+
+      class << self
+        def negotiate(env)
+          Response.new(
+            status: 403,
+            headers: { 'Content-Type' => 'text/html' },
+            body: [Datadog::AppSec::Assets.blocked(format: format(env))]
+          )
+        end
+
+        private
+
+        def format(env)
+          format = env['HTTP_ACCEPT'] && env['HTTP_ACCEPT'].split(',').any? do |accept|
+            if accept.start_with?('text/html')
+              break :html
+            elsif accept.start_with?('application/json')
+              break :json
+            end
+          end
+
+          format || :text
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/configuration/components.rb

@@ -248,6 +248,8 @@ module Datadog
             # NOTE: Please update the Initialization section of ProfilingDevelopment.md with any changes to this method
 
             if settings.profiling.advanced.force_enable_new_profiler
+              print_new_profiler_warnings
+
               recorder = Datadog::Profiling::StackRecorder.new
               collector = Datadog::Profiling::Collectors::CpuAndWallTimeWorker.new(
                 recorder: recorder,
@@ -331,20 +333,39 @@ module Datadog
           end
 
           def should_enable_gc_profiling?(settings)
-            return true if Gem::Version.new(RUBY_VERSION) < Gem::Version.new('3')
-
             # See comments on the setting definition for more context on why it exists.
             if settings.profiling.advanced.force_enable_gc_profiling
-              Datadog.logger.debug(
-                'Profiling time/resources spent in Garbage Collection force enabled. Do not use Ractors in combination ' \
-                'with this option as profiles will be incomplete.'
-              )
+              if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('3')
+                Datadog.logger.debug(
+                  'Profiling time/resources spent in Garbage Collection force enabled. Do not use Ractors in combination ' \
+                  'with this option as profiles will be incomplete.'
+                )
+              end
 
               true
             else
               false
             end
           end
+
+          def print_new_profiler_warnings
+            if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('2.6')
+              Datadog.logger.warn(
+                'New Ruby profiler has been force-enabled. This feature is in beta state. We do not yet recommend ' \
+                'running it in production environments. Please report any issues ' \
+                'you run into to Datadog support or via <https://github.com/datadog/dd-trace-rb/issues/new>!'
+              )
+            else
+              # For more details on the issue, see the "BIG Issue" comment on `gvl_owner` function in
+              # `private_vm_api_access.c`.
+              Datadog.logger.warn(
+                'New Ruby profiler has been force-enabled on a legacy Ruby version (< 2.6). This is not recommended in ' \
+                'production environments, as due to limitations in Ruby APIs, we suspect it may lead to crashes in very ' \
+                'rare situations. Please report any issues you run into to Datadog support or ' \
+                'via <https://github.com/datadog/dd-trace-rb/issues/new>!'
+              )
+            end
+          end
         end
 
         attr_reader \

data/lib/datadog/core/configuration/ext.rb

@@ -0,0 +1,18 @@
+# typed: true
+
+module Datadog
+  module Core
+    module Configuration
+      # Constants for configuration settings
+      # e.g. Env vars, default values, enums, etc...
+      module Ext
+        # @public_api
+        module Diagnostics
+          ENV_DEBUG_ENABLED = 'DD_TRACE_DEBUG'.freeze
+          ENV_HEALTH_METRICS_ENABLED = 'DD_HEALTH_METRICS_ENABLED'.freeze
+          ENV_STARTUP_LOGS_ENABLED = 'DD_TRACE_STARTUP_LOGS'.freeze
+        end
+      end
+    end
+  end
+end