ddtrace 1.11.1 → 1.12.1

data/lib/datadog/appsec/assets/waf_rules/strict.json

@@ -1,7 +1,7 @@
 {
   "version": "2.2",
   "metadata": {
-    "rules_version": "1.5.2"
+    "rules_version": "1.7.0"
   },
   "rules": [
     {
@@ -24,96 +24,51 @@
               }
             ],
             "list": [
-              "",
               "(hydra)",
-              ".nasl",
               "absinthe",
-              "advanced email extractor",
-              "arachni/",
               "autogetcontent",
               "bilbo",
               "bfac",
-              "brutus",
-              "brutus/aet",
-              "bsqlbf",
-              "cgichk",
               "cisco-torch",
-              "commix",
               "core-project/1.0",
               "crimscanner/",
               "datacha0s",
-              "detectify",
-              "dirbuster",
               "domino hunter",
               "dotdotpwn",
               "email extractor",
               "fhscan core 1.",
               "floodgate",
-              "fuzz faster u fool",
               "f-secure radar",
               "get-minimal",
-              "gobuster",
               "gootkit auto-rooter scanner",
               "grabber",
               "grendel-scan",
-              "havij",
               "inspath",
               "internet ninja",
-              "jaascois",
-              "jorgee",
               "masscan",
-              "metis",
               "morfeus fucking scanner",
               "mysqloit",
-              "n-stealth",
-              "nessus",
-              "netsparker",
-              "nikto",
-              "nmap nse",
-              "nmap scripting engine",
-              "nmap-nse",
-              "nsauditor",
-              "nuclei",
-              "openvas",
-              "pangolin",
-              "paros",
-              "pmafind",
               "prog.customcrawler",
               "qqgamehall",
-              "qualys was",
               "s.t.a.l.k.e.r.",
-              "security scan",
               "springenwerk",
               "sql power injector",
-              "sqlmap",
-              "sqlninja",
               "struts-pwn",
               "sysscan",
               "tbi-webscanner",
               "teh forest lobster",
-              "this is an exploit",
               "toata dragostea",
-              "toata dragostea mea pentru diavola",
               "uil2pn",
               "user-agent:",
               "vega/",
               "voideye",
-              "w3af.sf.net",
-              "w3af.sourceforge.net",
-              "w3af.org",
               "webbandit",
-              "webinspect",
               "webshag",
-              "webtrends security analyzer",
               "webvulnscan",
-              "wfuzz",
               "whatweb",
               "whcc/",
               "wordpress hash grabber",
-              "wpscan",
-              "xmlrpc exploit",
-              "zgrab",
-              "zmeu"
+              "xmlrpc exploit"
             ]
           },
           "operator": "phrase_match"

data/lib/datadog/appsec/configuration/settings.rb

@@ -116,7 +116,7 @@ module Datadog
         }.freeze
 
         # Struct constant whisker cast for Steep
-        Integration = _ = Struct.new(:integration, :options) # rubocop:disable Naming/ConstantName
+        Integration = _ = Struct.new(:integration) # rubocop:disable Naming/ConstantName
 
         def initialize
           @integrations = []
@@ -181,14 +181,6 @@ module Datadog
           _ = @options[:obfuscator_value_regex]
         end
 
-        def [](integration_name)
-          integration = Datadog::AppSec::Contrib::Integration.registry[integration_name]
-
-          raise ArgumentError, "'#{integration_name}' is not a valid integration." unless integration
-
-          integration.options
-        end
-
         def merge(dsl)
           dsl.options.each do |k, v|
             unless v.nil?
@@ -203,7 +195,7 @@ module Datadog
           dsl.instruments.each do |instrument|
             # TODO: error handling
             registered_integration = Datadog::AppSec::Contrib::Integration.registry[instrument.name]
-            @integrations << Integration.new(registered_integration, instrument.options)
+            @integrations << Integration.new(registered_integration)
 
             # TODO: move to a separate apply step
             klass = registered_integration.klass

data/lib/datadog/appsec/configuration.rb

@@ -15,7 +15,7 @@ module Datadog
       # Configuration DSL implementation
       class DSL
         # Struct constant whisker cast for Steep
-        Instrument = _ = Struct.new(:name, :options) # rubocop:disable Naming/ConstantName
+        Instrument = _ = Struct.new(:name) # rubocop:disable Naming/ConstantName
 
         def initialize
           @instruments = []
@@ -24,8 +24,8 @@ module Datadog
 
         attr_reader :instruments, :options
 
-        def instrument(name, options = {})
-          @instruments << Instrument.new(name, options)
+        def instrument(name)
+          @instruments << Instrument.new(name)
         end
 
         def enabled=(value)
@@ -64,12 +64,6 @@ module Datadog
         def obfuscator_value_regex=(value)
           options[:obfuscator_value_regex] = value
         end
-
-        def [](key)
-          found = @instruments.find { |e| e.name == key }
-
-          found.options if found
-        end
       end
 
       # class-level methods for Configuration

data/lib/datadog/appsec/contrib/rack/ext.rb

@@ -5,7 +5,6 @@ module Datadog
         # Rack integration constants
         module Ext
           APP = 'rack'.freeze
-          ENV_ENABLED = 'DD_TRACE_RACK_ENABLED'.freeze # TODO: DD_APPSEC?
         end
       end
     end

data/lib/datadog/appsec/contrib/rack/gateway/request.rb

@@ -25,14 +25,20 @@ module Datadog
             def query
               # Downstream libddwaf expects keys and values to be extractable
               # separately so we can't use [[k, v], ...]. We also want to allow
-              # duplicate keys, so we use [{k, v}, ...] instead.
-              request.query_string.split('&').map do |e|
+              # duplicate keys, so we use {k => [v, ...], ...} instead, taking into
+              # account that {k => [v1, v2, ...], ...} is possible for duplicate keys.
+              request.query_string.split('&').each.with_object({}) do |e, hash|
                 k, v = e.split('=').map { |s| CGI.unescape(s) }
+                hash[k] ||= []
 
-                { k => v }
+                hash[k] << v
               end
             end
 
+            def method
+              request.request_method
+            end
+
             def headers
               request.env.each_with_object({}) do |(k, v), h|
                 h[k.gsub(/^HTTP_/, '').downcase.tr('_', '-')] = v if k =~ /^HTTP_/
@@ -47,6 +53,14 @@ module Datadog
               request.url
             end
 
+            def fullpath
+              request.fullpath
+            end
+
+            def path
+              request.path
+            end
+
             def cookies
               request.cookies
             end

data/lib/datadog/appsec/contrib/rack/gateway/response.rb

@@ -9,14 +9,14 @@ module Datadog
         module Gateway
           # Gateway Response argument.
           class Response < Instrumentation::Gateway::Argument
-            attr_reader :body, :status, :headers, :active_context
+            attr_reader :body, :status, :headers, :scope
 
-            def initialize(body, status, headers, active_context:)
+            def initialize(body, status, headers, scope:)
               super()
               @body = body
               @status = status
               @headers = headers.each_with_object({}) { |(k, v), h| h[k.downcase] = v }
-              @active_context = active_context
+              @scope = scope
             end
 
             def response

data/lib/datadog/appsec/contrib/rack/gateway/watcher.rb

@@ -25,26 +25,26 @@ module Datadog
                 gateway.watch('rack.request', :appsec) do |stack, gateway_request|
                   block = false
                   event = nil
-                  waf_context = gateway_request.env['datadog.waf.context']
+                  scope = gateway_request.env[Datadog::AppSec::Ext::SCOPE_KEY]
 
                   AppSec::Reactive::Operation.new('rack.request') do |op|
-                    trace = active_trace
-                    span = active_span
-
-                    Rack::Reactive::Request.subscribe(op, waf_context) do |result, _block|
+                    Rack::Reactive::Request.subscribe(op, scope.processor_context) do |result, _block|
                       if result.status == :match
                         # TODO: should this hash be an Event instance instead?
                         event = {
                           waf_result: result,
-                          trace: trace,
-                          span: span,
+                          trace: scope.trace,
+                          span: scope.service_entry_span,
                           request: gateway_request,
                           actions: result.actions
                         }
 
-                        span.set_tag('appsec.event', 'true') if span
+                        if scope.service_entry_span
+                          scope.service_entry_span.set_tag('appsec.blocked', 'true') if result.actions.include?('block')
+                          scope.service_entry_span.set_tag('appsec.event', 'true')
+                        end
 
-                        waf_context.events << event
+                        scope.processor_context.events << event
                       end
                     end
 
@@ -68,26 +68,26 @@ module Datadog
                 gateway.watch('rack.response', :appsec) do |stack, gateway_response|
                   block = false
                   event = nil
-                  waf_context = gateway_response.active_context
+                  scope = gateway_response.scope
 
                   AppSec::Reactive::Operation.new('rack.response') do |op|
-                    trace = active_trace
-                    span = active_span
-
-                    Rack::Reactive::Response.subscribe(op, waf_context) do |result, _block|
+                    Rack::Reactive::Response.subscribe(op, scope.processor_context) do |result, _block|
                       if result.status == :match
                         # TODO: should this hash be an Event instance instead?
                         event = {
                           waf_result: result,
-                          trace: trace,
-                          span: span,
+                          trace: scope.trace,
+                          span: scope.service_entry_span,
                           response: gateway_response,
                           actions: result.actions
                         }
 
-                        span.set_tag('appsec.event', 'true') if span
+                        if scope.service_entry_span
+                          scope.service_entry_span.set_tag('appsec.blocked', 'true') if result.actions.include?('block')
+                          scope.service_entry_span.set_tag('appsec.event', 'true')
+                        end
 
-                        waf_context.events << event
+                        scope.processor_context.events << event
                       end
                     end
 
@@ -111,26 +111,26 @@ module Datadog
                 gateway.watch('rack.request.body', :appsec) do |stack, gateway_request|
                   block = false
                   event = nil
-                  waf_context = gateway_request.env['datadog.waf.context']
+                  scope = gateway_request.env[Datadog::AppSec::Ext::SCOPE_KEY]
 
                   AppSec::Reactive::Operation.new('rack.request.body') do |op|
-                    trace = active_trace
-                    span = active_span
-
-                    Rack::Reactive::RequestBody.subscribe(op, waf_context) do |result, _block|
+                    Rack::Reactive::RequestBody.subscribe(op, scope.processor_context) do |result, _block|
                       if result.status == :match
                         # TODO: should this hash be an Event instance instead?
                         event = {
                           waf_result: result,
-                          trace: trace,
-                          span: span,
+                          trace: scope.trace,
+                          span: scope.service_entry_span,
                           request: gateway_request,
                           actions: result.actions
                         }
 
-                        span.set_tag('appsec.event', 'true') if span
+                        if scope.service_entry_span
+                          scope.service_entry_span.set_tag('appsec.blocked', 'true') if result.actions.include?('block')
+                          scope.service_entry_span.set_tag('appsec.event', 'true')
+                        end
 
-                        waf_context.events << event
+                        scope.processor_context.events << event
                       end
                     end
 
@@ -149,24 +149,6 @@ module Datadog
                   [ret, res]
                 end
               end
-
-              private
-
-              def active_trace
-                # TODO: factor out tracing availability detection
-
-                return unless defined?(Datadog::Tracing)
-
-                Datadog::Tracing.active_trace
-              end
-
-              def active_span
-                # TODO: factor out tracing availability detection
-
-                return unless defined?(Datadog::Tracing)
-
-                Datadog::Tracing.active_span
-              end
             end
           end
         end

data/lib/datadog/appsec/contrib/rack/integration.rb

@@ -1,6 +1,5 @@
 require_relative '../integration'
 
-require_relative 'configuration/settings'
 require_relative 'patcher'
 require_relative 'request_middleware'
 require_relative 'request_body_middleware'
@@ -33,10 +32,6 @@ module Datadog
             false
           end
 
-          def default_configuration
-            Configuration::Settings.new
-          end
-
           def patcher
             Patcher
           end

data/lib/datadog/appsec/contrib/rack/reactive/request.rb

@@ -13,6 +13,7 @@ module Datadog
               'request.query',
               'request.cookies',
               'request.client_ip',
+              'server.request.method'
             ].freeze
             private_constant :ADDRESSES
 
@@ -20,14 +21,16 @@ module Datadog
               catch(:block) do
                 op.publish('request.query', gateway_request.query)
                 op.publish('request.headers', gateway_request.headers)
-                op.publish('request.uri.raw', gateway_request.url)
+                op.publish('request.uri.raw', gateway_request.fullpath)
                 op.publish('request.cookies', gateway_request.cookies)
                 op.publish('request.client_ip', gateway_request.client_ip)
+                op.publish('server.request.method', gateway_request.method)
 
                 nil
               end
             end
 
+            # rubocop:disable Metrics/MethodLength
             def self.subscribe(op, waf_context)
               op.subscribe(*ADDRESSES) do |*values|
                 Datadog.logger.debug { "reacted to #{ADDRESSES.inspect}: #{values.inspect}" }
@@ -37,6 +40,7 @@ module Datadog
                 query = values[2]
                 cookies = values[3]
                 client_ip = values[4]
+                request_method = values[5]
 
                 waf_args = {
                   'server.request.cookies' => cookies,
@@ -45,6 +49,7 @@ module Datadog
                   'server.request.headers' => headers,
                   'server.request.headers.no_cookies' => headers_no_cookies,
                   'http.client_ip' => client_ip,
+                  'server.request.method' => request_method,
                 }
 
                 waf_timeout = Datadog::AppSec.settings.waf_timeout
@@ -71,6 +76,7 @@ module Datadog
                   Datadog.logger.debug { "WAF UNKNOWN: #{result.status.inspect} #{result.inspect}" }
                 end
               end
+              # rubocop:enable Metrics/MethodLength
             end
           end
         end

data/lib/datadog/appsec/contrib/rack/request_body_middleware.rb

@@ -17,7 +17,7 @@ module Datadog
           end
 
           def call(env)
-            context = env['datadog.waf.context']
+            context = env[Datadog::AppSec::Ext::SCOPE_KEY]
 
             return @app.call(env) unless context
 

data/lib/datadog/appsec/contrib/rack/request_middleware.rb

@@ -2,7 +2,6 @@ require 'json'
 
 require_relative 'gateway/request'
 require_relative 'gateway/response'
-require_relative '../../ext'
 require_relative '../../instrumentation/gateway'
 require_relative '../../processor'
 require_relative '../../response'
@@ -23,7 +22,7 @@ module Datadog
             @oneshot_tags_sent = false
           end
 
-          # rubocop:disable Metrics/PerceivedComplexity,Metrics/CyclomaticComplexity,Metrics/MethodLength
+          # rubocop:disable Metrics/AbcSize,Metrics/PerceivedComplexity,Metrics/CyclomaticComplexity,Metrics/MethodLength
           def call(env)
             return @app.call(env) unless Datadog::AppSec.enabled?
 
@@ -31,14 +30,19 @@ module Datadog
 
             processor = nil
             ready = false
-            context = nil
+            scope = nil
+
+            # For a given request, keep using the first Rack stack scope for
+            # nested apps. Don't set `context` local variable so that on popping
+            # out of this nested stack we don't finalize the parent's context
+            return @app.call(env) if active_scope(env)
 
             Datadog::AppSec.reconfigure_lock do
               processor = Datadog::AppSec.processor
 
               if !processor.nil? && processor.ready?
-                context = processor.activate_context
-                env['datadog.waf.context'] = context
+                scope = Datadog::AppSec::Scope.activate_scope(active_trace, active_span, processor)
+                env[Datadog::AppSec::Ext::SCOPE_KEY] = scope
                 ready = true
               end
             end
@@ -49,7 +53,7 @@ module Datadog
 
             gateway_request = Gateway::Request.new(env)
 
-            add_appsec_tags(processor, active_trace, active_span, env)
+            add_appsec_tags(processor, scope.trace, scope.service_entry_span, env)
 
             request_return, request_response = catch(::Datadog::AppSec::Ext::INTERRUPT) do
               Instrumentation.gateway.push('rack.request', gateway_request) do
@@ -65,17 +69,17 @@ module Datadog
               request_return[2],
               request_return[0],
               request_return[1],
-              active_context: context
+              scope: scope
             )
 
             _response_return, response_response = Instrumentation.gateway.push('rack.response', gateway_response)
 
-            context.events.each do |e|
+            scope.processor_context.events.each do |e|
               e[:response] ||= gateway_response
               e[:request]  ||= gateway_request
             end
 
-            AppSec::Event.record(*context.events)
+            AppSec::Event.record(scope.service_entry_span, *scope.processor_context.events)
 
             if response_response && response_response.any? { |action, _event| action == :block }
               request_return = AppSec::Response.negotiate(env).to_rack
@@ -83,15 +87,19 @@ module Datadog
 
             request_return
           ensure
-            if context
-              add_waf_runtime_tags(active_trace, context)
-              processor.deactivate_context
+            if scope
+              add_waf_runtime_tags(scope.service_entry_span, scope.processor_context)
+              Datadog::AppSec::Scope.deactivate_scope
             end
           end
-          # rubocop:enable Metrics/PerceivedComplexity,Metrics/CyclomaticComplexity,Metrics/MethodLength
+          # rubocop:enable Metrics/AbcSize,Metrics/PerceivedComplexity,Metrics/CyclomaticComplexity,Metrics/MethodLength
 
           private
 
+          def active_scope(env)
+            env[Datadog::AppSec::Ext::SCOPE_KEY]
+          end
+
           def active_trace
             # TODO: factor out tracing availability detection
 
@@ -111,9 +119,9 @@ module Datadog
           def add_appsec_tags(processor, trace, span, env)
             return unless trace
 
-            trace.set_tag('_dd.appsec.enabled', 1)
-            trace.set_tag('_dd.runtime_family', 'ruby')
-            trace.set_tag('_dd.appsec.waf.version', Datadog::AppSec::WAF::VERSION::BASE_STRING)
+            span.set_tag('_dd.appsec.enabled', 1)
+            span.set_tag('_dd.runtime_family', 'ruby')
+            span.set_tag('_dd.appsec.waf.version', Datadog::AppSec::WAF::VERSION::BASE_STRING)
 
             if span && span.get_tag(Tracing::Metadata::Ext::HTTP::TAG_CLIENT_IP).nil?
               request_header_collection = Datadog::Tracing::Contrib::Rack::Header::RequestHeaderCollection.new(env)
@@ -127,17 +135,17 @@ module Datadog
             end
 
             if processor.ruleset_info
-              trace.set_tag('_dd.appsec.event_rules.version', processor.ruleset_info[:version])
+              span.set_tag('_dd.appsec.event_rules.version', processor.ruleset_info[:version])
 
               unless @oneshot_tags_sent
                 # Small race condition, but it's inoccuous: worst case the tags
                 # are sent a couple of times more than expected
                 @oneshot_tags_sent = true
 
-                trace.set_tag('_dd.appsec.event_rules.loaded', processor.ruleset_info[:loaded].to_f)
-                trace.set_tag('_dd.appsec.event_rules.error_count', processor.ruleset_info[:failed].to_f)
-                trace.set_tag('_dd.appsec.event_rules.errors', JSON.dump(processor.ruleset_info[:errors]))
-                trace.set_tag('_dd.appsec.event_rules.addresses', JSON.dump(processor.addresses))
+                span.set_tag('_dd.appsec.event_rules.loaded', processor.ruleset_info[:loaded].to_f)
+                span.set_tag('_dd.appsec.event_rules.error_count', processor.ruleset_info[:failed].to_f)
+                span.set_tag('_dd.appsec.event_rules.errors', JSON.dump(processor.ruleset_info[:errors]))
+                span.set_tag('_dd.appsec.event_rules.addresses', JSON.dump(processor.addresses))
 
                 # Ensure these tags reach the backend
                 trace.keep!
@@ -149,15 +157,15 @@ module Datadog
             end
           end
 
-          def add_waf_runtime_tags(trace, context)
-            return unless trace
+          def add_waf_runtime_tags(span, context)
+            return unless span
             return unless context
 
-            trace.set_tag('_dd.appsec.waf.timeouts', context.timeouts)
+            span.set_tag('_dd.appsec.waf.timeouts', context.timeouts)
 
             # these tags expect time in us
-            trace.set_tag('_dd.appsec.waf.duration', context.time_ns / 1000.0)
-            trace.set_tag('_dd.appsec.waf.duration_ext', context.time_ext_ns / 1000.0)
+            span.set_tag('_dd.appsec.waf.duration', context.time_ns / 1000.0)
+            span.set_tag('_dd.appsec.waf.duration_ext', context.time_ext_ns / 1000.0)
           end
         end
       end

data/lib/datadog/appsec/contrib/rails/ext.rb

@@ -5,7 +5,6 @@ module Datadog
         # Rack integration constants
         module Ext
           APP = 'rails'.freeze
-          ENV_ENABLED = 'DD_TRACE_RAILS_ENABLED'.freeze
         end
       end
     end

data/lib/datadog/appsec/contrib/rails/framework.rb

@@ -8,21 +8,9 @@ module Datadog
         module Framework
           def self.setup
             Datadog::AppSec.configure do |datadog_config|
-              rails_config = config_with_defaults(datadog_config)
-              activate_rack!(datadog_config, rails_config)
+              datadog_config.instrument(:rack)
             end
           end
-
-          def self.config_with_defaults(datadog_config)
-            datadog_config[:rails]
-          end
-
-          # Apply relevant configuration from Sinatra to Rack
-          def self.activate_rack!(datadog_config, sinatra_config)
-            datadog_config.instrument(
-              :rack,
-            )
-          end
         end
       end
     end