ddtrace 1.0.0 → 1.9.0

data/lib/datadog/appsec/processor.rb

@@ -1,28 +1,57 @@
 # typed: ignore
 
-require 'datadog/appsec/assets'
+require_relative 'assets'
 
 module Datadog
   module AppSec
     # Processor integrates libddwaf into datadog/appsec
     class Processor
-      # Interface object to check using case .. when
-      module IOLike
-        def read; end
-        def rewind; end
+      # Context manages a sequence of runs
+      class Context
+        attr_reader :time_ns, :time_ext_ns, :timeouts, :events
+
+        def initialize(processor)
+          @context = Datadog::AppSec::WAF::Context.new(processor.send(:handle))
+          @time_ns = 0.0
+          @time_ext_ns = 0.0
+          @timeouts = 0
+          @events = []
+        end
+
+        def run(input, timeout = WAF::LibDDWAF::DDWAF_RUN_TIMEOUT)
+          start_ns = Core::Utils::Time.get_time(:nanosecond)
+
+          # TODO: remove multiple assignment
+          _code, res = _ = @context.run(input, timeout)
+          # @type var res: WAF::Result
+
+          stop_ns = Core::Utils::Time.get_time(:nanosecond)
+
+          @time_ns += res.total_runtime
+          @time_ext_ns += (stop_ns - start_ns)
+          @timeouts += 1 if res.timeout
+
+          res
+        end
 
-        def self.===(other)
-          instance_methods.all? { |meth| other.respond_to?(meth) }
+        def finalize
+          @context.finalize
         end
       end
 
+      attr_reader :ruleset_info, :addresses
+
       def initialize
-        @ruleset = nil
-        @handle = nil
+        @ruleset_info = nil
+        @addresses = []
 
         unless load_libddwaf && load_ruleset && create_waf_handle
           Datadog.logger.warn { 'AppSec is disabled, see logged errors above' }
+
+          return
         end
+
+        update_ip_denylist
       end
 
       def ready?
@@ -30,9 +59,39 @@ module Datadog
       end
 
       def new_context
-        Datadog::AppSec::WAF::Context.new(@handle)
+        Context.new(self)
+      end
+
+      def update_rule_data(data)
+        @handle.update_rule_data(data)
+      end
+
+      def toggle_rules(map)
+        @handle.toggle_rules(map)
       end
 
+      def update_ip_denylist(denylist = Datadog::AppSec.settings.ip_denylist, id: 'blocked_ips')
+        denylist ||= []
+
+        ruledata_setting = [
+          {
+            'id' => id,
+            'type' => 'data_with_expiration',
+            'data' => denylist.map { |ip| { 'value' => ip.to_s, 'expiration' => 2**63 } }
+          }
+        ]
+
+        update_rule_data(ruledata_setting)
+      end
+
+      def finalize
+        @handle.finalize
+      end
+
+      protected
+
+      attr_reader :handle
+
       private
 
       def load_libddwaf
@@ -48,8 +107,8 @@ module Datadog
                        JSON.parse(Datadog::AppSec::Assets.waf_rules(ruleset_setting))
                      when String
                        JSON.parse(File.read(ruleset_setting))
-                     when IOLike
-                       JSON.parse(ruleset_setting.read).tap { ruleset_setting.rewind }
+                     when File, StringIO
+                       JSON.parse(ruleset_setting.read || '').tap { ruleset_setting.rewind }
                      when Hash
                        ruleset_setting
                      else
@@ -70,9 +129,23 @@ module Datadog
         # TODO: this may need to be reset if the main Datadog logging level changes after initialization
         Datadog::AppSec::WAF.logger = Datadog.logger if Datadog.logger.debug? && Datadog::AppSec.settings.waf_debug
 
-        @handle = Datadog::AppSec::WAF::Handle.new(@ruleset)
+        obfuscator_config = {
+          key_regex: Datadog::AppSec.settings.obfuscator_key_regex,
+          value_regex: Datadog::AppSec.settings.obfuscator_value_regex,
+        }
+        @handle = Datadog::AppSec::WAF::Handle.new(@ruleset, obfuscator: obfuscator_config)
+        @ruleset_info = @handle.ruleset_info
+        @addresses = @handle.required_addresses
 
         true
+      rescue WAF::LibDDWAF::Error => e
+        Datadog.logger.error do
+          "libddwaf failed to initialize, error: #{e.inspect}"
+        end
+
+        @ruleset_info = e.ruleset_info if e.ruleset_info
+
+        false
       rescue StandardError => e
         Datadog.logger.error do
           "libddwaf failed to initialize, error: #{e.inspect}"

data/lib/datadog/appsec/rate_limiter.rb

@@ -1,4 +1,4 @@
-# typed: true
+# typed: ignore
 
 module Datadog
   module AppSec
@@ -35,6 +35,11 @@ module Datadog
           rate_limiter(name).limit(&block)
         end
 
+        # reset a rate limiter: used for testing
+        def reset!(name)
+          Thread.current[:datadog_security_trace_rate_limiter] = nil
+        end
+
         protected
 
         def rate_limiter(name)

data/lib/datadog/appsec/reactive/address_hash.rb

@@ -1,4 +1,4 @@
-# typed: true
+# typed: ignore
 
 module Datadog
   module AppSec

data/lib/datadog/appsec/reactive/engine.rb

@@ -1,7 +1,7 @@
-# typed: true
+# typed: ignore
 
-require 'datadog/appsec/reactive/address_hash'
-require 'datadog/appsec/reactive/subscriber'
+require_relative 'address_hash'
+require_relative 'subscriber'
 
 module Datadog
   module AppSec

data/lib/datadog/appsec/reactive/operation.rb

@@ -1,7 +1,6 @@
-# typed: true
+# typed: ignore
 
-require 'datadog/appsec'
-require 'datadog/appsec/reactive/engine'
+require_relative 'engine'
 
 module Datadog
   module AppSec
@@ -9,14 +8,14 @@ module Datadog
       # Reactive Engine nested operation tracking
       class Operation
         attr_reader :reactive,
-                    :parent,
-                    :name
+          :parent,
+          :name
 
         def initialize(name, parent = nil, reactive_engine = nil)
           Datadog.logger.debug { "operation: #{name} initialize" }
           @name = name
           @parent = parent
-          @reactive = reactive_engine || parent && parent.reactive || Reactive::Engine.new
+          @reactive = reactive_engine || (parent.reactive unless parent.nil?) || Reactive::Engine.new
 
           # TODO: concurrent store
           # TODO: constant

data/lib/datadog/appsec/reactive/subscriber.rb

@@ -1,4 +1,4 @@
-# typed: true
+# typed: ignore
 
 module Datadog
   module AppSec

data/lib/datadog/appsec/response.rb

@@ -0,0 +1,63 @@
+# typed: ignore
+
+require_relative 'assets'
+require_relative 'utils/http/media_range'
+
+module Datadog
+  module AppSec
+    # AppSec response
+    class Response
+      attr_reader :status, :headers, :body
+
+      def initialize(status:, headers: {}, body: [])
+        @status = status
+        @headers = headers
+        @body = body
+      end
+
+      def to_rack
+        [status, headers, body]
+      end
+
+      def to_sinatra_response
+        ::Sinatra::Response.new(body, status, headers)
+      end
+
+      def to_action_dispatch_response
+        ::ActionDispatch::Response.new(status, headers, body)
+      end
+
+      class << self
+        def negotiate(env)
+          Response.new(
+            status: 403,
+            headers: { 'Content-Type' => 'text/html' },
+            body: [Datadog::AppSec::Assets.blocked(format: format(env))]
+          )
+        end
+
+        private
+
+        FORMAT_MAP = {
+          'text/html' => :html,
+          'application/json' => :json,
+          'text/plain' => :text,
+        }.freeze
+
+        DEFAULT_FORMAT = :text
+
+        def format(env)
+          return DEFAULT_FORMAT unless env.key?('HTTP_ACCEPT')
+
+          accepted = env['HTTP_ACCEPT'].split(',').map { |m| Utils::HTTP::MediaRange.new(m) }.sort
+
+          accepted.each_with_object(DEFAULT_FORMAT) do |_default, range|
+            format = FORMAT_MAP.keys.find { |type, _format| range === type }
+
+            return FORMAT_MAP[format] if format
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/utils/http/media_range.rb

@@ -0,0 +1,201 @@
+# typed: ignore
+
+require_relative 'media_type'
+
+module Datadog
+  module AppSec
+    module Utils
+      module HTTP
+        # Implementation of media range for content negotiation
+        class MediaRange
+          class ParseError < ::StandardError
+          end
+
+          WILDCARD = '*'.freeze
+          WILDCARD_RE = ::Regexp.escape(WILDCARD)
+
+          # See: https://www.rfc-editor.org/rfc/rfc7230#section-3.2.6
+          TOKEN_RE = /[-#$%&'*+.^_`|~A-Za-z0-9]+/.freeze
+
+          # See: https://www.rfc-editor.org/rfc/rfc7231#section-3.1.1.1
+          PARAMETER_RE = %r{ # rubocop:disable Style/RegexpLiteral
+            (?:
+              (?<parameter_name>#{TOKEN_RE})
+              =
+              (?:
+                (?<parameter_value>#{TOKEN_RE})
+                |
+                "(?<parameter_value>[^"]+)"
+              )
+            )
+          }ix.freeze
+
+          # See: https://www.rfc-editor.org/rfc/rfc7231#section-5.3.2
+          ACCEPT_EXT_RE = %r{ # rubocop:disable Style/RegexpLiteral
+            (?:
+              (?<ext_name>#{TOKEN_RE})
+              =
+              (?:
+                (?<ext_value>#{TOKEN_RE})
+                |
+                "(?<ext_value>[^"]+)"
+              )
+            )
+          }ix.freeze
+
+          # See: https://www.rfc-editor.org/rfc/rfc7231#section-5.3.1
+          QVALUE_RE = %r{ # rubocop:disable Style/RegexpLiteral
+            0(?:\.\d{1,3})?
+            |
+            1(?:\.0{1,3})?
+          }ix.freeze
+
+          # See: https://www.rfc-editor.org/rfc/rfc7231#section-5.3.2
+          MEDIA_RANGE_RE = %r{
+            \A
+            (?:
+              (?<type>#{WILDCARD_RE})/(?<subtype>#{WILDCARD_RE})
+              |
+              (?<type>#{TOKEN_RE})/(?<subtype>#{WILDCARD_RE})
+              |
+              (?<type>#{TOKEN_RE})/(?<subtype>#{TOKEN_RE})
+            )
+            (?<parameters>
+              (?:
+                \s*;\s*
+                (?!q=)
+                #{PARAMETER_RE}
+              )*
+            )
+            (?<accept_params>
+              (?<weight>
+                \s*;\s*
+                (?:q=
+                  (?<quality>
+                    #{QVALUE_RE}
+                  )
+                )
+              )
+              (?<accept_exts>
+                (?<accept_ext>
+                  (?:
+                    \s*;\s*
+                    (?!q=)
+                    #{ACCEPT_EXT_RE}
+                  )*
+                )
+              )
+            )?
+            \Z
+          }ix.freeze
+
+          attr_reader :type, :subtype, :quality, :parameters, :accept_ext
+
+          def initialize(media_range) # rubocop:disable Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity
+            media_range_match = MEDIA_RANGE_RE.match(media_range)
+
+            raise ParseError, media_range.inspect if media_range_match.nil?
+
+            @type = (media_range_match['type'] || WILDCARD).downcase
+            @subtype = (media_range_match['subtype'] || WILDCARD).downcase
+            @quality = (media_range_match['quality'] || 1.0).to_f
+            @parameters = {}
+            @accept_ext = {}
+
+            parameters = media_range_match['parameters']
+
+            return if parameters.nil?
+
+            parameters.split(';').map(&:strip).each do |parameter|
+              parameter_match = PARAMETER_RE.match(parameter)
+
+              next if parameter_match.nil?
+
+              parameter_name = parameter_match['parameter_name']
+              parameter_value = parameter_match['parameter_value']
+
+              next if parameter_name.nil? || parameter_value.nil?
+
+              @parameters[parameter_name.downcase] = parameter_value.downcase
+            end
+
+            accept_exts = media_range_match['accept_exts']
+
+            return if accept_exts.nil?
+
+            accept_exts.split(';').map(&:strip).each do |ext|
+              ext_match = ACCEPT_EXT_RE.match(ext)
+
+              next if ext_match.nil?
+
+              ext_name = ext_match['ext_name']
+              ext_value = ext_match['ext_value']
+
+              next if ext_name.nil? || ext_value.nil?
+
+              @accept_ext[ext_name.downcase] = ext_value.downcase
+            end
+          end
+
+          # Compare two MediaRange for ordering
+          def <=>(other)
+            unless (q = quality <=> other.quality) == 0 || q.nil?
+              return q
+            end
+
+            if (s = specificity <=> other.specificity) != 0
+              return s
+            end
+
+            unless wildcard?(:type)
+              if wildcard?(:subtype) && !other.wildcard?(:subtype)
+                return -1
+              elsif !wildcard?(:subtype) && other.wildcard?(:subtype)
+                return 1
+              end
+            end
+
+            if wildcard?(:type) && !other.wildcard?(:type)
+              return -1
+            elsif !wildcard?(:type) && other.wildcard?(:type)
+              return 1
+            end
+
+            0
+          end
+
+          # Compare with a MediaType for match
+          #
+          # returns true if the MediaType is accepted by this MediaRange
+          def ===(other)
+            return self === MediaType.new(other) if other.is_a?(::String)
+
+            type == other.type && subtype == other.subtype && other.parameters.all? { |k, v| parameters[k] == v } ||
+              type == other.type && wildcard?(:subtype) ||
+              wildcard?(:type) && wildcard?(:subtype)
+          end
+
+          def specificity
+            @parameters.count
+          end
+
+          def wildcard?(field = nil)
+            return wildcard?(:type) || wildcard?(:subtype) if field.nil?
+
+            instance_variable_get(:"@#{field}") == WILDCARD
+          end
+
+          def to_s
+            s = "#{@type}/#{@subtype}"
+
+            s << ';' << @parameters.map { |k, v| "#{k}=#{v}" }.join(';') if @parameters.count > 0
+            s << ";q=#{@quality}" if @quality < 1.0
+            s << ';' << @accept_ext.map { |k, v| "#{k}=#{v}" }.join(';') if @accept_ext.count > 0
+
+            s
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/utils/http/media_type.rb

@@ -0,0 +1,87 @@
+# typed: ignore
+
+module Datadog
+  module AppSec
+    module Utils
+      module HTTP
+        # Implementation of media type for content negotiation
+        #
+        # See:
+        # - https://www.rfc-editor.org/rfc/rfc7231#section-5.3.1
+        # - https://www.rfc-editor.org/rfc/rfc7231#section-5.3.2
+        class MediaType
+          class ParseError < ::StandardError
+          end
+
+          WILDCARD = '*'.freeze
+
+          # See: https://www.rfc-editor.org/rfc/rfc7230#section-3.2.6
+          TOKEN_RE = /[-#$%&'*+.^_`|~A-Za-z0-9]+/.freeze
+
+          # See: https://www.rfc-editor.org/rfc/rfc7231#section-3.1.1.1
+          PARAMETER_RE = %r{ # rubocop:disable Style/RegexpLiteral
+            (?:
+              (?<parameter_name>#{TOKEN_RE})
+              =
+              (?:
+                (?<parameter_value>#{TOKEN_RE})
+                |
+                "(?<parameter_value>[^"]+)"
+              )
+            )
+          }ix.freeze
+
+          # See: https://www.rfc-editor.org/rfc/rfc7231#section-3.1.1.1
+          MEDIA_TYPE_RE = %r{
+            \A
+            (?<type>#{TOKEN_RE})/(?<subtype>#{TOKEN_RE})
+            (?<parameters>
+              (?:
+                \s*;\s*
+                #{PARAMETER_RE}
+              )*
+            )
+            \Z
+          }ix.freeze
+
+          attr_reader :type, :subtype, :parameters
+
+          def initialize(media_type)
+            media_type_match = MEDIA_TYPE_RE.match(media_type)
+
+            raise ParseError, media_type.inspect if media_type_match.nil?
+
+            @type = (media_type_match['type'] || WILDCARD).downcase
+            @subtype = (media_type_match['subtype'] || WILDCARD).downcase
+            @parameters = {}
+
+            parameters = media_type_match['parameters']
+
+            return if parameters.nil?
+
+            parameters.split(';').map(&:strip).each do |parameter|
+              parameter_match = PARAMETER_RE.match(parameter)
+
+              next if parameter_match.nil?
+
+              parameter_name = parameter_match['parameter_name']
+              parameter_value = parameter_match['parameter_value']
+
+              next if parameter_name.nil? || parameter_value.nil?
+
+              @parameters[parameter_name.downcase] = parameter_value.downcase
+            end
+          end
+
+          def to_s
+            s = "#{@type}/#{@subtype}"
+
+            s << ';' << @parameters.map { |k, v| "#{k}=#{v}" }.join(';') if @parameters.count > 0
+
+            s
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/utils/http.rb

@@ -0,0 +1,9 @@
+module Datadog
+  module AppSec
+    module Utils
+      # HTTP utilities
+      module HTTP
+      end
+    end
+  end
+end

data/lib/datadog/appsec/utils.rb

@@ -0,0 +1,7 @@
+module Datadog
+  module AppSec
+    # Utilities for AppSec
+    module Utils
+    end
+  end
+end

data/lib/datadog/appsec.rb

@@ -1,7 +1,7 @@
-# typed: false
+# typed: ignore
 
-require 'datadog/appsec/configuration'
-require 'datadog/appsec/extensions'
+require_relative 'appsec/configuration'
+require_relative 'appsec/extensions'
 
 module Datadog
   # Namespace for Datadog AppSec instrumentation
@@ -18,6 +18,6 @@ module Datadog
 end
 
 # Integrations
-require 'datadog/appsec/contrib/rack/integration'
-require 'datadog/appsec/contrib/sinatra/integration'
-require 'datadog/appsec/contrib/rails/integration'
+require_relative 'appsec/contrib/rack/integration'
+require_relative 'appsec/contrib/sinatra/integration'
+require_relative 'appsec/contrib/rails/integration'

data/lib/datadog/ci/configuration/components.rb

@@ -1,6 +1,6 @@
 # typed: true
 
-require 'datadog/ci/flush'
+require_relative '../flush'
 
 module Datadog
   module CI

data/lib/datadog/ci/configuration/settings.rb

@@ -1,6 +1,6 @@
 # typed: false
 
-require 'datadog/ci/ext/settings'
+require_relative '../ext/settings'
 
 module Datadog
   module CI

data/lib/datadog/ci/contrib/cucumber/configuration/settings.rb

@@ -1,7 +1,7 @@
 # typed: false
 
-require 'datadog/tracing/contrib/configuration/settings'
-require 'datadog/ci/contrib/cucumber/ext'
+require_relative '../../../../tracing/contrib/configuration/settings'
+require_relative '../ext'
 
 module Datadog
   module CI

data/lib/datadog/ci/contrib/cucumber/formatter.rb

@@ -1,10 +1,10 @@
 # typed: true
 
-require 'datadog/ci/test'
-require 'datadog/ci/ext/app_types'
-require 'datadog/ci/ext/environment'
-require 'datadog/ci/ext/test'
-require 'datadog/ci/contrib/cucumber/ext'
+require_relative '../../test'
+require_relative '../../ext/app_types'
+require_relative '../../ext/environment'
+require_relative '../../ext/test'
+require_relative 'ext'
 
 module Datadog
   module CI

data/lib/datadog/ci/contrib/cucumber/instrumentation.rb

@@ -1,6 +1,6 @@
 # typed: true
 
-require 'datadog/ci/contrib/cucumber/formatter'
+require_relative 'formatter'
 
 module Datadog
   module CI

data/lib/datadog/ci/contrib/cucumber/integration.rb

@@ -1,9 +1,9 @@
 # typed: false
 
-require 'datadog/tracing/contrib/integration'
+require_relative '../../../tracing/contrib/integration'
 
-require 'datadog/ci/contrib/cucumber/configuration/settings'
-require 'datadog/ci/contrib/cucumber/patcher'
+require_relative 'configuration/settings'
+require_relative 'patcher'
 
 module Datadog
   module CI
@@ -35,7 +35,7 @@ module Datadog
             false
           end
 
-          def default_configuration
+          def new_configuration
             Configuration::Settings.new
           end