ddtrace 1.0.0 → 1.9.0

data/lib/datadog/appsec/assets.rb

@@ -1,4 +1,4 @@
-# typed: false
+# typed: ignore
 
 require 'pathname'
 
@@ -12,8 +12,8 @@ module Datadog
         read("waf_rules/#{kind}.json")
       end
 
-      def blocked
-        @blocked ||= read('blocked.html')
+      def blocked(format: :html)
+        (@blocked ||= {})[format] ||= read("blocked.#{format}")
       end
 
       def path

data/lib/datadog/appsec/autoload.rb

@@ -1,16 +1,18 @@
 # typed: ignore
 
 if %w[1 true].include?((ENV['DD_APPSEC_ENABLED'] || '').downcase)
+  require_relative '../../ddtrace'
+
   begin
-    require 'datadog/appsec'
+    require_relative '../appsec'
   rescue StandardError => e
-    puts "AppSec failed to load. No security check will be performed. error: #{e.message}"
+    puts "AppSec failed to load. No security check will be performed. error: #{e.class.name} #{e.message}"
   end
 
   begin
-    require 'datadog/appsec/contrib/auto_instrument'
+    require_relative 'contrib/auto_instrument'
     Datadog::AppSec::Contrib::AutoInstrument.patch_all
   rescue StandardError => e
-    puts "AppSec failed to instrument. No security check will be performed. error: #{e.message}"
+    puts "AppSec failed to instrument. No security check will be performed. error: #{e.class.name} #{e.message}"
   end
 end

data/lib/datadog/appsec/configuration/settings.rb

@@ -1,4 +1,4 @@
-# typed: false
+# typed: ignore
 
 module Datadog
   module AppSec
@@ -8,7 +8,8 @@ module Datadog
       class Settings
         class << self
           def boolean
-            lambda do |v|
+            # @type ^(::String) -> bool
+            ->(v) do # rubocop:disable Style/Lambda
               case v
               when /(1|true)/i
                 true
@@ -22,14 +23,16 @@ module Datadog
 
           # TODO: allow symbols
           def string
+            # @type ^(::String) -> ::String
             ->(v) { v.to_s }
           end
 
           def integer
-            lambda do |v|
+            # @type ^(::String) -> ::Integer
+            ->(v) do # rubocop:disable Style/Lambda
               case v
               when /(\d+)/
-                Integer(Regexp.last_match[1])
+                Regexp.last_match(1).to_i
               else
                 raise ArgumentError, "invalid integer: #{v.inspect}"
               end
@@ -38,7 +41,8 @@ module Datadog
 
           # rubocop:disable Metrics/MethodLength
           def duration(base = :ns, type = :integer)
-            lambda do |v|
+            # @type ^(::String) -> ::Integer | ::Float
+            ->(v) do # rubocop:disable Style/Lambda
               cast = case type
                      when :integer, Integer
                        method(:Integer)
@@ -63,19 +67,19 @@ module Datadog
 
               case v
               when /^(\d+)h$/
-                cast.call(Regexp.last_match[1]) * 1_000_000_000 * 60 * 60 / scale
+                cast.call(Regexp.last_match(1)) * 1_000_000_000 * 60 * 60 / scale
               when /^(\d+)m$/
-                cast.call(Regexp.last_match[1]) * 1_000_000_000 * 60 / scale
+                cast.call(Regexp.last_match(1)) * 1_000_000_000 * 60 / scale
               when /^(\d+)s$/
-                cast.call(Regexp.last_match[1]) * 1_000_000_000 / scale
+                cast.call(Regexp.last_match(1)) * 1_000_000_000 / scale
               when /^(\d+)ms$/
-                cast.call(Regexp.last_match[1]) * 1_000_000 / scale
+                cast.call(Regexp.last_match(1)) * 1_000_000 / scale
               when /^(\d+)us$/
-                cast.call(Regexp.last_match[1]) * 1_000 / scale
+                cast.call(Regexp.last_match(1)) * 1_000 / scale
               when /^(\d+)ns$/
-                cast.call(Regexp.last_match[1]) / scale
+                cast.call(Regexp.last_match(1)) / scale
               when /^(\d+)$/
-                cast.call(Regexp.last_match[1])
+                cast.call(Regexp.last_match(1))
               else
                 raise ArgumentError, "invalid duration: #{v.inspect}"
               end
@@ -84,12 +88,19 @@ module Datadog
           # rubocop:enable Metrics/MethodLength
         end
 
+        # rubocop:disable Layout/LineLength
+        DEFAULT_OBFUSCATOR_KEY_REGEX = '(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?)key)|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)|bearer|authorization'.freeze
+        DEFAULT_OBFUSCATOR_VALUE_REGEX = '(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?|access_?|secret_?)key(?:_?id)?|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)(?:\s*=[^;]|"\s*:\s*"[^"]+")|bearer\s+[a-z0-9\._\-]+|token:[a-z0-9]{13}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L][\w=-]+\.ey[I-L][\w=-]+(?:\.[\w.+\/=-]+)?|[\-]{5}BEGIN[a-z\s]+PRIVATE\sKEY[\-]{5}[^\-]+[\-]{5}END[a-z\s]+PRIVATE\sKEY|ssh-rsa\s*[a-z0-9\/\.+]{100,}'.freeze
+        # rubocop:enable Layout/LineLength
+
         DEFAULTS = {
           enabled: false,
           ruleset: :recommended,
           waf_timeout: 5_000, # us
           waf_debug: false,
           trace_rate_limit: 100, # traces/s
+          obfuscator_key_regex: DEFAULT_OBFUSCATOR_KEY_REGEX,
+          obfuscator_value_regex: DEFAULT_OBFUSCATOR_VALUE_REGEX,
         }.freeze
 
         ENVS = {
@@ -98,9 +109,12 @@ module Datadog
           'DD_APPSEC_WAF_TIMEOUT' => [:waf_timeout, Settings.duration(:us)],
           'DD_APPSEC_WAF_DEBUG' => [:waf_debug, Settings.boolean],
           'DD_APPSEC_TRACE_RATE_LIMIT' => [:trace_rate_limit, Settings.integer],
+          'DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP' => [:obfuscator_key_regex, Settings.string],
+          'DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP' => [:obfuscator_value_regex, Settings.string],
         }.freeze
 
-        Integration = Struct.new(:integration, :options)
+        # Struct constant whisker cast for Steep
+        Integration = _ = Struct.new(:integration, :options) # rubocop:disable Naming/ConstantName
 
         def initialize
           @integrations = []
@@ -112,23 +126,45 @@ module Datadog
         end
 
         def enabled
-          @options[:enabled]
+          # Cast for Steep
+          _ = @options[:enabled]
         end
 
         def ruleset
-          @options[:ruleset]
+          # Cast for Steep
+          _ = @options[:ruleset]
+        end
+
+        # EXPERIMENTAL: This configurable is not meant to be publicly used, but
+        #               is very useful for testing. It may change at any point in time.
+        def ip_denylist
+          # Cast for Steep
+          _ = @options[:ip_denylist]
         end
 
         def waf_timeout
-          @options[:waf_timeout]
+          # Cast for Steep
+          _ = @options[:waf_timeout]
         end
 
         def waf_debug
-          @options[:waf_debug]
+          # Cast for Steep
+          _ = @options[:waf_debug]
         end
 
         def trace_rate_limit
-          @options[:trace_rate_limit]
+          # Cast for Steep
+          _ = @options[:trace_rate_limit]
+        end
+
+        def obfuscator_key_regex
+          # Cast for Steep
+          _ = @options[:obfuscator_key_regex]
+        end
+
+        def obfuscator_value_regex
+          # Cast for Steep
+          _ = @options[:obfuscator_value_regex]
         end
 
         def [](integration_name)
@@ -136,7 +172,7 @@ module Datadog
 
           raise ArgumentError, "'#{integration_name}' is not a valid integration." unless integration
 
-          integration.options if integration
+          integration.options
         end
 
         def merge(dsl)

data/lib/datadog/appsec/configuration.rb

@@ -1,6 +1,6 @@
-# typed: true
+# typed: ignore
 
-require 'datadog/appsec/configuration/settings'
+require_relative 'configuration/settings'
 
 module Datadog
   module AppSec
@@ -14,7 +14,9 @@ module Datadog
 
       # Configuration DSL implementation
       class DSL
-        Instrument = Struct.new(:name, :options)
+        # Struct constant whisker cast for Steep
+        Instrument = _ = Struct.new(:name, :options) # rubocop:disable Naming/ConstantName
+
         def initialize
           @instruments = []
           @options = {}
@@ -34,6 +36,10 @@ module Datadog
           options[:ruleset] = value
         end
 
+        def ip_denylist=(value)
+          options[:ip_denylist] = value
+        end
+
         # in microseconds
         def waf_timeout=(value)
           options[:waf_timeout] = value
@@ -47,6 +53,14 @@ module Datadog
           options[:trace_rate_limit] = value
         end
 
+        def obfuscator_key_regex=(value)
+          options[:obfuscator_key_regex] = value
+        end
+
+        def obfuscator_value_regex=(value)
+          options[:obfuscator_value_regex] = value
+        end
+
         def [](key)
           found = @instruments.find { |e| e.name == key }
 

data/lib/datadog/appsec/contrib/auto_instrument.rb

@@ -1,6 +1,4 @@
-# typed: true
-
-require 'ddtrace'
+# typed: ignore
 
 module Datadog
   module AppSec

data/lib/datadog/appsec/contrib/configuration/settings.rb

@@ -1,6 +1,6 @@
-# typed: false
+# typed: ignore
 
-require 'datadog/core/configuration/base'
+require_relative '../../../core/configuration/base'
 
 module Datadog
   module AppSec

data/lib/datadog/appsec/contrib/integration.rb

@@ -1,4 +1,4 @@
-# typed: true
+# typed: ignore
 
 module Datadog
   module AppSec

data/lib/datadog/appsec/contrib/patcher.rb

@@ -1,4 +1,4 @@
-# typed: strict
+# typed: ignore
 
 module Datadog
   module AppSec

data/lib/datadog/appsec/contrib/rack/configuration/settings.rb

@@ -1,7 +1,7 @@
-# typed: false
+# typed: ignore
 
-require 'datadog/appsec/contrib/configuration/settings'
-require 'datadog/appsec/contrib/rack/ext'
+require_relative '../../configuration/settings'
+require_relative '../ext'
 
 module Datadog
   module AppSec

data/lib/datadog/appsec/contrib/rack/ext.rb

@@ -1,4 +1,4 @@
-# typed: true
+# typed: ignore
 
 module Datadog
   module AppSec

data/lib/datadog/appsec/contrib/rack/gateway/watcher.rb

@@ -1,10 +1,11 @@
-# typed: false
+# typed: ignore
 
-require 'datadog/appsec/instrumentation/gateway'
-require 'datadog/appsec/reactive/operation'
-require 'datadog/appsec/contrib/rack/reactive/request'
-require 'datadog/appsec/contrib/rack/reactive/response'
-require 'datadog/appsec/event'
+require_relative '../../../instrumentation/gateway'
+require_relative '../../../reactive/operation'
+require_relative '../reactive/request'
+require_relative '../reactive/request_body'
+require_relative '../reactive/response'
+require_relative '../../../event'
 
 module Datadog
   module AppSec
@@ -14,44 +15,37 @@ module Datadog
           # Watcher for Rack gateway events
           module Watcher
             # rubocop:disable Metrics/AbcSize
-            # rubocop:disable Metrics/CyclomaticComplexity
             # rubocop:disable Metrics/MethodLength
+            # rubocop:disable Metrics/CyclomaticComplexity
             # rubocop:disable Metrics/PerceivedComplexity
             def self.watch
-              Instrumentation.gateway.watch('rack.request') do |stack, request|
+              Instrumentation.gateway.watch('rack.request', :appsec) do |stack, request|
                 block = false
                 event = nil
                 waf_context = request.env['datadog.waf.context']
 
                 AppSec::Reactive::Operation.new('rack.request') do |op|
-                  # TODO: factor out
-                  if defined?(Datadog::Tracing) && Datadog::Tracing.respond_to?(:active_span)
-                    active_trace = Datadog::Tracing.active_trace
-                    active_span = Datadog::Tracing.active_span
+                  trace = active_trace
+                  span = active_span
 
-                    Datadog.logger.debug { "active span: #{active_span.span_id}" } if active_span
-
-                    if active_span
-                      active_span.set_tag('_dd.appsec.enabled', 1)
-                      active_span.set_tag('_dd.runtime_family', 'ruby')
-                    end
-                  end
-
-                  Rack::Reactive::Request.subscribe(op, waf_context) do |action, result, _block|
-                    record = [:block, :monitor].include?(action)
-                    if record
+                  Rack::Reactive::Request.subscribe(op, waf_context) do |result, _block|
+                    if result.status == :match
                       # TODO: should this hash be an Event instance instead?
                       event = {
                         waf_result: result,
-                        trace: active_trace,
-                        span: active_span,
+                        trace: trace,
+                        span: span,
                         request: request,
-                        action: action
+                        actions: result.actions
                       }
+
+                      span.set_tag('appsec.event', 'true') if span
+
+                      waf_context.events << event
                     end
                   end
 
-                  _action, _result, block = Rack::Reactive::Request.publish(op, request)
+                  _result, block = Rack::Reactive::Request.publish(op, request)
                 end
 
                 next [nil, [[:block, event]]] if block
@@ -66,45 +60,79 @@ module Datadog
                 [ret, res]
               end
 
-              Instrumentation.gateway.watch('rack.response') do |stack, response|
+              Instrumentation.gateway.watch('rack.response', :appsec) do |stack, response|
                 block = false
                 event = nil
                 waf_context = response.instance_eval { @waf_context }
 
                 AppSec::Reactive::Operation.new('rack.response') do |op|
-                  # TODO: factor out
-                  if defined?(Datadog::Tracing) && Datadog::Tracing.respond_to?(:active_span)
-                    active_trace = Datadog::Tracing.active_trace
-                    active_span = Datadog::Tracing.active_span
+                  trace = active_trace
+                  span = active_span
+
+                  Rack::Reactive::Response.subscribe(op, waf_context) do |result, _block|
+                    if result.status == :match
+                      # TODO: should this hash be an Event instance instead?
+                      event = {
+                        waf_result: result,
+                        trace: trace,
+                        span: span,
+                        response: response,
+                        actions: result.actions
+                      }
 
-                    Datadog.logger.debug { "active span: #{active_span.span_id}" } if active_span
+                      span.set_tag('appsec.event', 'true') if span
 
-                    if active_span
-                      active_span.set_tag('_dd.appsec.enabled', 1)
-                      active_span.set_tag('_dd.runtime_family', 'ruby')
+                      waf_context.events << event
                     end
                   end
 
-                  Rack::Reactive::Response.subscribe(op, waf_context) do |action, result, _block|
-                    record = [:block, :monitor].include?(action)
-                    if record
+                  _result, block = Rack::Reactive::Response.publish(op, response)
+                end
+
+                next [nil, [[:block, event]]] if block
+
+                ret, res = stack.call(response)
+
+                if event
+                  res ||= []
+                  res << [:monitor, event]
+                end
+
+                [ret, res]
+              end
+
+              Instrumentation.gateway.watch('rack.request.body', :appsec) do |stack, request|
+                block = false
+                event = nil
+                waf_context = request.env['datadog.waf.context']
+
+                AppSec::Reactive::Operation.new('rack.request.body') do |op|
+                  trace = active_trace
+                  span = active_span
+
+                  Rack::Reactive::RequestBody.subscribe(op, waf_context) do |result, _block|
+                    if result.status == :match
                       # TODO: should this hash be an Event instance instead?
                       event = {
                         waf_result: result,
-                        trace: active_trace,
-                        span: active_span,
-                        response: response,
-                        action: action
+                        trace: trace,
+                        span: span,
+                        request: request,
+                        actions: result.actions
                       }
+
+                      span.set_tag('appsec.event', 'true') if span
+
+                      waf_context.events << event
                     end
                   end
 
-                  _action, _result, block = Rack::Reactive::Response.publish(op, response)
+                  _result, block = Rack::Reactive::RequestBody.publish(op, request)
                 end
 
                 next [nil, [[:block, event]]] if block
 
-                ret, res = stack.call(response)
+                ret, res = stack.call(request)
 
                 if event
                   res ||= []
@@ -114,10 +142,30 @@ module Datadog
                 [ret, res]
               end
             end
-            # rubocop:enable Metrics/AbcSize
             # rubocop:enable Metrics/CyclomaticComplexity
-            # rubocop:enable Metrics/MethodLength
             # rubocop:enable Metrics/PerceivedComplexity
+            # rubocop:enable Metrics/MethodLength
+            # rubocop:enable Metrics/AbcSize
+
+            class << self
+              private
+
+              def active_trace
+                # TODO: factor out tracing availability detection
+
+                return unless defined?(Datadog::Tracing)
+
+                Datadog::Tracing.active_trace
+              end
+
+              def active_span
+                # TODO: factor out tracing availability detection
+
+                return unless defined?(Datadog::Tracing)
+
+                Datadog::Tracing.active_span
+              end
+            end
           end
         end
       end

data/lib/datadog/appsec/contrib/rack/integration.rb

@@ -1,10 +1,11 @@
 # typed: ignore
 
-require 'datadog/appsec/contrib/integration'
+require_relative '../integration'
 
-require 'datadog/appsec/contrib/rack/configuration/settings'
-require 'datadog/appsec/contrib/rack/patcher'
-require 'datadog/appsec/contrib/rack/request_middleware'
+require_relative 'configuration/settings'
+require_relative 'patcher'
+require_relative 'request_middleware'
+require_relative 'request_body_middleware'
 
 module Datadog
   module AppSec

data/lib/datadog/appsec/contrib/rack/patcher.rb

@@ -1,8 +1,7 @@
 # typed: ignore
 
-require 'datadog/appsec/contrib/patcher'
-require 'datadog/appsec/contrib/rack/integration'
-require 'datadog/appsec/contrib/rack/gateway/watcher'
+require_relative '../patcher'
+require_relative 'gateway/watcher'
 
 module Datadog
   module AppSec

data/lib/datadog/appsec/contrib/rack/reactive/request.rb

@@ -1,6 +1,6 @@
-# typed: true
+# typed: ignore
 
-require 'datadog/appsec/contrib/rack/request'
+require_relative '../request'
 
 module Datadog
   module AppSec
@@ -15,9 +15,7 @@ module Datadog
                 op.publish('request.headers', Rack::Request.headers(request))
                 op.publish('request.uri.raw', Rack::Request.url(request))
                 op.publish('request.cookies', Rack::Request.cookies(request))
-                # op.publish('request.body.raw', Rack::Request.body(request))
-                # TODO: op.publish('request.path_params', { k: v }) # route params only?
-                # TODO: op.publish('request.path', request.script_name + request.path) # unused for now
+                op.publish('request.client_ip', Rack::Request.client_ip(request))
 
                 nil
               end
@@ -30,8 +28,7 @@ module Datadog
                 'request.uri.raw',
                 'request.query',
                 'request.cookies',
-                # 'request.body.raw',
-                # TODO: 'request.path_params',
+                'request.client_ip',
               ]
 
               op.subscribe(*addresses) do |*values|
@@ -41,40 +38,39 @@ module Datadog
                 uri_raw = values[1]
                 query = values[2]
                 cookies = values[3]
-                # body = values[4]
+                client_ip = values[4]
 
                 waf_args = {
                   'server.request.cookies' => cookies,
-                  # 'server.request.body.raw' => body,
                   'server.request.query' => query,
                   'server.request.uri.raw' => uri_raw,
                   'server.request.headers' => headers,
                   'server.request.headers.no_cookies' => headers_no_cookies,
-                  # TODO: 'server.request.path_params' => path_params,
+                  'http.client_ip' => client_ip,
                 }
 
                 waf_timeout = Datadog::AppSec.settings.waf_timeout
-                action, result = waf_context.run(waf_args, waf_timeout)
+                result = waf_context.run(waf_args, waf_timeout)
 
                 Datadog.logger.debug { "WAF TIMEOUT: #{result.inspect}" } if result.timeout
 
-                # TODO: encapsulate return array in a type
-                case action
-                when :monitor
+                case result.status
+                when :match
                   Datadog.logger.debug { "WAF: #{result.inspect}" }
-                  yield [action, result, false]
-                when :block
-                  Datadog.logger.debug { "WAF: #{result.inspect}" }
-                  yield [action, result, true]
-                  throw(:block, [action, result, true])
-                when :good
+
+                  block = result.actions.include?('block')
+
+                  yield [result, block]
+
+                  throw(:block, [result, true]) if block
+                when :ok
                   Datadog.logger.debug { "WAF OK: #{result.inspect}" }
                 when :invalid_call
                   Datadog.logger.debug { "WAF CALL ERROR: #{result.inspect}" }
                 when :invalid_rule, :invalid_flow, :no_rule
                   Datadog.logger.debug { "WAF RULE ERROR: #{result.inspect}" }
                 else
-                  Datadog.logger.debug { "WAF UNKNOWN: #{action.inspect} #{result.inspect}" }
+                  Datadog.logger.debug { "WAF UNKNOWN: #{result.status.inspect} #{result.inspect}" }
                 end
               end
             end