ddtrace 0.49.0 → 1.12.1

data/lib/datadog/appsec/assets.rb

@@ -0,0 +1,36 @@
+require 'pathname'
+
+module Datadog
+  module AppSec
+    # Helper methods to get vendored assets
+    module Assets
+      module_function
+
+      def waf_rules(kind = :recommended)
+        read("waf_rules/#{kind}.json")
+      end
+
+      def blocked(format: :html)
+        (@blocked ||= {})[format] ||= read("blocked.#{format}")
+      end
+
+      def path
+        Pathname.new(dir).join('assets')
+      end
+
+      def filepath(filename)
+        path.join(filename)
+      end
+
+      def read(filename, mode = 'rb')
+        File.open(filepath(filename), mode) { |f| f.read || raise('Unexpected nil IO object') }
+      end
+
+      def dir
+        # Happens only if this file is evaluated standalone, which should not happen
+        # Necessary to make type-checker happy with a non-nilable return value
+        __dir__ || raise('Unexpected file eval')
+      end
+    end
+  end
+end

data/lib/datadog/appsec/autoload.rb

@@ -0,0 +1,11 @@
+if %w[1 true].include?((ENV['DD_APPSEC_ENABLED'] || '').downcase)
+  begin
+    require_relative 'contrib/auto_instrument'
+    Datadog::AppSec::Contrib::AutoInstrument.patch_all
+  rescue StandardError => e
+    Kernel.warn(
+      '[ddtrace] AppSec failed to instrument. No security check will be performed. error: ' \
+      " #{e.class.name} #{e.message}"
+    )
+  end
+end

data/lib/datadog/appsec/component.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require_relative 'processor'
+require_relative 'processor/rule_merger'
+require_relative 'processor/rule_loader'
+
+module Datadog
+  module AppSec
+    # Core-pluggable component for AppSec
+    class Component
+      class << self
+        def build_appsec_component(settings)
+          return unless settings.respond_to?(:appsec) && settings.appsec.enabled
+
+          processor = create_processor(settings)
+          new(processor: processor)
+        end
+
+        private
+
+        def create_processor(settings)
+          rules = AppSec::Processor::RuleLoader.load_rules(ruleset: settings.appsec.ruleset)
+          return nil unless rules
+
+          data = AppSec::Processor::RuleLoader.load_data(
+            ip_denylist: settings.appsec.ip_denylist,
+            user_id_denylist: settings.appsec.user_id_denylist
+          )
+
+          ruleset = AppSec::Processor::RuleMerger.merge(
+            rules: [rules],
+            data: data,
+          )
+
+          processor = Processor.new(ruleset: ruleset)
+          return nil unless processor.ready?
+
+          processor
+        end
+      end
+
+      attr_reader :processor
+
+      def initialize(processor:)
+        @processor = processor
+        @mutex = Mutex.new
+      end
+
+      def reconfigure(ruleset:)
+        @mutex.synchronize do
+          new = Processor.new(ruleset: ruleset)
+
+          if new && new.ready?
+            old = @processor
+            @processor = new
+            old.finalize if old
+          end
+        end
+      end
+
+      def reconfigure_lock(&block)
+        @mutex.synchronize(&block)
+      end
+
+      def shutdown!
+        @mutex.synchronize do
+          if processor && processor.ready?
+            processor.finalize
+            @processor = nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/configuration/settings.rb

@@ -0,0 +1,224 @@
+# frozen_string_literal: true
+
+require 'set'
+
+module Datadog
+  module AppSec
+    module Configuration
+      # Configuration settings, acting as an integration registry
+      # TODO: as with Configuration, this is a trivial implementation
+      class Settings
+        class << self
+          def boolean
+            # @type ^(::String) -> bool
+            ->(v) do # rubocop:disable Style/Lambda
+              case v
+              when /(1|true)/i
+                true
+              when /(0|false)/i, nil
+                false
+              else
+                raise ArgumentError, "invalid boolean: #{v.inspect}"
+              end
+            end
+          end
+
+          # TODO: allow symbols
+          def string
+            # @type ^(::String) -> ::String
+            ->(v) { v.to_s }
+          end
+
+          def integer
+            # @type ^(::String) -> ::Integer
+            ->(v) do # rubocop:disable Style/Lambda
+              case v
+              when /(\d+)/
+                Regexp.last_match(1).to_i
+              else
+                raise ArgumentError, "invalid integer: #{v.inspect}"
+              end
+            end
+          end
+
+          # rubocop:disable Metrics/MethodLength
+          def duration(base = :ns, type = :integer)
+            # @type ^(::String) -> ::Integer | ::Float
+            ->(v) do # rubocop:disable Style/Lambda
+              cast = case type
+                     when :integer, Integer
+                       method(:Integer)
+                     when :float, Float
+                       method(:Float)
+                     else
+                       raise ArgumentError, "invalid type: #{v.inspect}"
+                     end
+
+              scale = case base
+                      when :s
+                        1_000_000_000
+                      when :ms
+                        1_000_000
+                      when :us
+                        1000
+                      when :ns
+                        1
+                      else
+                        raise ArgumentError, "invalid base: #{v.inspect}"
+                      end
+
+              case v
+              when /^(\d+)h$/
+                cast.call(Regexp.last_match(1)) * 1_000_000_000 * 60 * 60 / scale
+              when /^(\d+)m$/
+                cast.call(Regexp.last_match(1)) * 1_000_000_000 * 60 / scale
+              when /^(\d+)s$/
+                cast.call(Regexp.last_match(1)) * 1_000_000_000 / scale
+              when /^(\d+)ms$/
+                cast.call(Regexp.last_match(1)) * 1_000_000 / scale
+              when /^(\d+)us$/
+                cast.call(Regexp.last_match(1)) * 1_000 / scale
+              when /^(\d+)ns$/
+                cast.call(Regexp.last_match(1)) / scale
+              when /^(\d+)$/
+                cast.call(Regexp.last_match(1))
+              else
+                raise ArgumentError, "invalid duration: #{v.inspect}"
+              end
+            end
+          end
+          # rubocop:enable Metrics/MethodLength
+        end
+
+        # rubocop:disable Layout/LineLength
+        DEFAULT_OBFUSCATOR_KEY_REGEX = '(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?)key)|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)|bearer|authorization'
+        DEFAULT_OBFUSCATOR_VALUE_REGEX = '(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?|access_?|secret_?)key(?:_?id)?|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)(?:\s*=[^;]|"\s*:\s*"[^"]+")|bearer\s+[a-z0-9\._\-]+|token:[a-z0-9]{13}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L][\w=-]+\.ey[I-L][\w=-]+(?:\.[\w.+\/=-]+)?|[\-]{5}BEGIN[a-z\s]+PRIVATE\sKEY[\-]{5}[^\-]+[\-]{5}END[a-z\s]+PRIVATE\sKEY|ssh-rsa\s*[a-z0-9\/\.+]{100,}'
+        # rubocop:enable Layout/LineLength
+
+        DEFAULTS = {
+          enabled: false,
+          ruleset: :recommended,
+          waf_timeout: 5_000, # us
+          waf_debug: false,
+          trace_rate_limit: 100, # traces/s
+          obfuscator_key_regex: DEFAULT_OBFUSCATOR_KEY_REGEX,
+          obfuscator_value_regex: DEFAULT_OBFUSCATOR_VALUE_REGEX,
+        }.freeze
+
+        ENVS = {
+          'DD_APPSEC_ENABLED' => [:enabled, Settings.boolean],
+          'DD_APPSEC_RULES' => [:ruleset, Settings.string],
+          'DD_APPSEC_WAF_TIMEOUT' => [:waf_timeout, Settings.duration(:us)],
+          'DD_APPSEC_WAF_DEBUG' => [:waf_debug, Settings.boolean],
+          'DD_APPSEC_TRACE_RATE_LIMIT' => [:trace_rate_limit, Settings.integer],
+          'DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP' => [:obfuscator_key_regex, Settings.string],
+          'DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP' => [:obfuscator_value_regex, Settings.string],
+        }.freeze
+
+        # Struct constant whisker cast for Steep
+        Integration = _ = Struct.new(:integration) # rubocop:disable Naming/ConstantName
+
+        def initialize
+          @integrations = []
+          # Stores which options have been configured using Datadog.configure block or ENV variables
+          @configured = Set.new
+          @options = DEFAULTS.dup.tap do |options|
+            ENVS.each do |env, (key, conv)|
+              if ENV[env]
+                options[key] = conv.call(ENV[env])
+                @configured << key
+              end
+            end
+          end
+        end
+
+        def enabled
+          # Cast for Steep
+          _ = @options[:enabled]
+        end
+
+        def ruleset
+          # Cast for Steep
+          _ = @options[:ruleset]
+        end
+
+        # EXPERIMENTAL: This configurable is not meant to be publicly used, but
+        #               is very useful for testing. It may change at any point in time.
+        def ip_denylist
+          # Cast for Steep
+          _ = @options[:ip_denylist] || []
+        end
+
+        # EXPERIMENTAL: This configurable is not meant to be publicly used, but
+        #               is very useful for testing. It may change at any point in time.
+        def user_id_denylist
+          # Cast for Steep
+          _ = @options[:user_id_denylist] || []
+        end
+
+        def waf_timeout
+          # Cast for Steep
+          _ = @options[:waf_timeout]
+        end
+
+        def waf_debug
+          # Cast for Steep
+          _ = @options[:waf_debug]
+        end
+
+        def trace_rate_limit
+          # Cast for Steep
+          _ = @options[:trace_rate_limit]
+        end
+
+        def obfuscator_key_regex
+          # Cast for Steep
+          _ = @options[:obfuscator_key_regex]
+        end
+
+        def obfuscator_value_regex
+          # Cast for Steep
+          _ = @options[:obfuscator_value_regex]
+        end
+
+        def merge(dsl)
+          dsl.options.each do |k, v|
+            unless v.nil?
+              @options[k] = v
+              @configured << k
+            end
+          end
+
+          return self unless @options[:enabled]
+
+          # patcher.patch may call configure again, hence merge might be called again so it needs to be reentrant
+          dsl.instruments.each do |instrument|
+            # TODO: error handling
+            registered_integration = Datadog::AppSec::Contrib::Integration.registry[instrument.name]
+            @integrations << Integration.new(registered_integration)
+
+            # TODO: move to a separate apply step
+            klass = registered_integration.klass
+            if klass.loaded? && klass.compatible?
+              instance = klass.new
+              instance.patcher.patch
+            end
+          end
+
+          self
+        end
+
+        private
+
+        def default?(option)
+          !@configured.include?(option)
+        end
+
+        # Restore to original state, for testing only.
+        def reset!
+          initialize
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/configuration.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require_relative 'configuration/settings'
+
+module Datadog
+  module AppSec
+    # Configuration for AppSec
+    # TODO: this is a trivial implementation, check with shareable code with
+    # tracer and other products
+    module Configuration
+      def self.included(base)
+        base.extend(ClassMethods)
+      end
+
+      # Configuration DSL implementation
+      class DSL
+        # Struct constant whisker cast for Steep
+        Instrument = _ = Struct.new(:name) # rubocop:disable Naming/ConstantName
+
+        def initialize
+          @instruments = []
+          @options = {}
+        end
+
+        attr_reader :instruments, :options
+
+        def instrument(name)
+          @instruments << Instrument.new(name)
+        end
+
+        def enabled=(value)
+          options[:enabled] = value
+        end
+
+        def ruleset=(value)
+          options[:ruleset] = value
+        end
+
+        def ip_denylist=(value)
+          options[:ip_denylist] = value
+        end
+
+        def user_id_denylist=(value)
+          options[:user_id_denylist] = value
+        end
+
+        # in microseconds
+        def waf_timeout=(value)
+          options[:waf_timeout] = value
+        end
+
+        def waf_debug=(value)
+          options[:waf_debug] = value
+        end
+
+        def trace_rate_limit=(value)
+          options[:trace_rate_limit] = value
+        end
+
+        def obfuscator_key_regex=(value)
+          options[:obfuscator_key_regex] = value
+        end
+
+        def obfuscator_value_regex=(value)
+          options[:obfuscator_value_regex] = value
+        end
+      end
+
+      # class-level methods for Configuration
+      module ClassMethods
+        def configure
+          dsl = DSL.new
+          yield dsl
+          settings.merge(dsl)
+          settings
+        end
+
+        def settings
+          @settings ||= Settings.new
+        end
+
+        private
+
+        def default_setting?(setting)
+          settings.send(:default?, setting)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/auto_instrument.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Contrib
+      # Auto-instrumentation for security integrations
+      # TODO: this implementation is trivial, check for shareable code with tracer
+      module AutoInstrument
+        def self.patch_all
+          integrations = []
+
+          Datadog::AppSec::Contrib::Integration.registry.each do |_name, integration|
+            next unless integration.klass.auto_instrument?
+
+            integrations << integration.name
+          end
+
+          Datadog::AppSec.configure do |c|
+            integrations.each do |integration_name|
+              c.instrument integration_name
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/integration.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Contrib
+      # Base provides features that are shared across all integrations
+      module Integration
+        @registry = {}
+
+        RegisteredIntegration = Struct.new(:name, :klass, :options)
+
+        def self.included(base)
+          base.extend(ClassMethods)
+        end
+
+        # Class-level methods for Integration
+        module ClassMethods
+          def register_as(name, options = {})
+            Integration.register(self, name, options)
+          end
+
+          def compatible?
+            true
+          end
+        end
+
+        def self.register(integration, name, options)
+          registry[name] = RegisteredIntegration.new(name, integration, options)
+        end
+
+        def self.registry
+          @registry
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/patcher.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Contrib
+      # Common patcher code for AppSec integrations
+      # TODO: empty implementation, check with tracer shareable code
+      module Patcher
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/rack/ext.rb

@@ -0,0 +1,12 @@
+module Datadog
+  module AppSec
+    module Contrib
+      module Rack
+        # Rack integration constants
+        module Ext
+          APP = 'rack'.freeze
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/rack/gateway/request.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+require_relative '../../../instrumentation/gateway/argument'
+require_relative '../../../../core/header_collection'
+require_relative '../../../../tracing/client_ip'
+
+module Datadog
+  module AppSec
+    module Contrib
+      module Rack
+        module Gateway
+          # Gateway Request argument. Normalized extration of data from Rack::Request
+          class Request < Instrumentation::Gateway::Argument
+            attr_reader :env
+
+            def initialize(env)
+              super()
+              @env = env
+            end
+
+            def request
+              @request ||= ::Rack::Request.new(env)
+            end
+
+            def query
+              # Downstream libddwaf expects keys and values to be extractable
+              # separately so we can't use [[k, v], ...]. We also want to allow
+              # duplicate keys, so we use {k => [v, ...], ...} instead, taking into
+              # account that {k => [v1, v2, ...], ...} is possible for duplicate keys.
+              request.query_string.split('&').each.with_object({}) do |e, hash|
+                k, v = e.split('=').map { |s| CGI.unescape(s) }
+                hash[k] ||= []
+
+                hash[k] << v
+              end
+            end
+
+            def method
+              request.request_method
+            end
+
+            def headers
+              request.env.each_with_object({}) do |(k, v), h|
+                h[k.gsub(/^HTTP_/, '').downcase.tr('_', '-')] = v if k =~ /^HTTP_/
+              end
+            end
+
+            def body
+              request.body.read.tap { request.body.rewind }
+            end
+
+            def url
+              request.url
+            end
+
+            def fullpath
+              request.fullpath
+            end
+
+            def path
+              request.path
+            end
+
+            def cookies
+              request.cookies
+            end
+
+            def host
+              request.host
+            end
+
+            def user_agent
+              request.user_agent
+            end
+
+            def remote_addr
+              env['REMOTE_ADDR']
+            end
+
+            def form_hash
+              # force form data processing
+              request.POST if request.form_data?
+
+              # usually Hash<String,String> but can be a more complex
+              # Hash<String,String||Array||Hash> when e.g coming from JSON
+              env['rack.request.form_hash']
+            end
+
+            def client_ip
+              remote_ip = remote_addr
+              header_collection = Datadog::Core::HeaderCollection.from_hash(headers)
+
+              Datadog::Tracing::ClientIp.extract_client_ip(header_collection, remote_ip)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/rack/gateway/response.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require_relative '../../../instrumentation/gateway/argument'
+
+module Datadog
+  module AppSec
+    module Contrib
+      module Rack
+        module Gateway
+          # Gateway Response argument.
+          class Response < Instrumentation::Gateway::Argument
+            attr_reader :body, :status, :headers, :scope
+
+            def initialize(body, status, headers, scope:)
+              super()
+              @body = body
+              @status = status
+              @headers = headers.each_with_object({}) { |(k, v), h| h[k.downcase] = v }
+              @scope = scope
+            end
+
+            def response
+              @response ||= ::Rack::Response.new(body, status, headers)
+            end
+          end
+        end
+      end
+    end
+  end
+end