ddtrace 0.54.2 → 1.8.0

data/lib/datadog/appsec/processor.rb

@@ -0,0 +1,198 @@
+# typed: ignore
+
+require_relative 'assets'
+
+module Datadog
+  module AppSec
+    # Processor integrates libddwaf into datadog/appsec
+    class Processor
+      # Interface object to check using case .. when
+      module IOLike
+        def read; end
+        def rewind; end
+
+        def self.===(other)
+          instance_methods.all? { |meth| other.respond_to?(meth) }
+        end
+      end
+
+      # Context manages a sequence of runs
+      class Context
+        attr_reader :time_ns, :time_ext_ns, :timeouts, :events
+
+        def initialize(processor)
+          @context = Datadog::AppSec::WAF::Context.new(processor.send(:handle))
+          @time_ns = 0.0
+          @time_ext_ns = 0.0
+          @timeouts = 0
+          @events = []
+        end
+
+        def run(*args)
+          start_ns = Core::Utils::Time.get_time(:nanosecond)
+
+          _code, res = @context.run(*args)
+
+          stop_ns = Core::Utils::Time.get_time(:nanosecond)
+
+          @time_ns += res.total_runtime
+          @time_ext_ns += (stop_ns - start_ns)
+          @timeouts += 1 if res.timeout
+
+          res
+        end
+
+        def finalize
+          @context.finalize
+        end
+      end
+
+      attr_reader :ruleset_info, :addresses
+
+      def initialize
+        @ruleset = nil
+        @handle = nil
+        @ruleset_info = nil
+        @addresses = nil
+
+        unless load_libddwaf && load_ruleset && create_waf_handle
+          Datadog.logger.warn { 'AppSec is disabled, see logged errors above' }
+
+          return
+        end
+
+        update_ip_denylist
+      end
+
+      def ready?
+        !@ruleset.nil? && !@handle.nil?
+      end
+
+      def new_context
+        Context.new(self)
+      end
+
+      def update_rule_data(data)
+        @handle.update_rule_data(data)
+      end
+
+      def toggle_rules(map)
+        @handle.toggle_rules(map)
+      end
+
+      def update_ip_denylist(denylist = Datadog::AppSec.settings.ip_denylist, id: 'blocked_ips')
+        denylist ||= []
+
+        ruledata_setting = [
+          {
+            'id' => id,
+            'type' => 'data_with_expiration',
+            'data' => denylist.map { |ip| { 'value' => ip.to_s, 'expiration' => 2**63 } }
+          }
+        ]
+
+        update_rule_data(ruledata_setting)
+      end
+
+      def finalize
+        @handle.finalize
+      end
+
+      protected
+
+      attr_reader :handle
+
+      private
+
+      def load_libddwaf
+        Processor.require_libddwaf && Processor.libddwaf_provides_waf?
+      end
+
+      def load_ruleset
+        ruleset_setting = Datadog::AppSec.settings.ruleset
+
+        begin
+          @ruleset = case ruleset_setting
+                     when :recommended, :risky, :strict
+                       JSON.parse(Datadog::AppSec::Assets.waf_rules(ruleset_setting))
+                     when String
+                       JSON.parse(File.read(ruleset_setting))
+                     when IOLike
+                       JSON.parse(ruleset_setting.read).tap { ruleset_setting.rewind }
+                     when Hash
+                       ruleset_setting
+                     else
+                       raise ArgumentError, "unsupported value for ruleset setting: #{ruleset_setting.inspect}"
+                     end
+
+          true
+        rescue StandardError => e
+          Datadog.logger.error do
+            "libddwaf ruleset failed to load, ruleset: #{ruleset_setting.inspect} error: #{e.inspect}"
+          end
+
+          false
+        end
+      end
+
+      def create_waf_handle
+        # TODO: this may need to be reset if the main Datadog logging level changes after initialization
+        Datadog::AppSec::WAF.logger = Datadog.logger if Datadog.logger.debug? && Datadog::AppSec.settings.waf_debug
+
+        obfuscator_config = {
+          key_regex: Datadog::AppSec.settings.obfuscator_key_regex,
+          value_regex: Datadog::AppSec.settings.obfuscator_value_regex,
+        }
+        @handle = Datadog::AppSec::WAF::Handle.new(@ruleset, obfuscator: obfuscator_config)
+        @ruleset_info = @handle.ruleset_info
+        @addresses = @handle.required_addresses
+
+        true
+      rescue StandardError => e
+        Datadog.logger.error do
+          "libddwaf failed to initialize, error: #{e.inspect}"
+        end
+
+        @ruleset_info = e.ruleset_info if e.respond_to?(:ruleset_info)
+
+        false
+      end
+
+      class << self
+        # check whether libddwaf is required *and* able to provide the needed feature
+        def libddwaf_provides_waf?
+          defined?(Datadog::AppSec::WAF) ? true : false
+        end
+
+        # libddwaf raises a LoadError on unsupported platforms; it may at some
+        # point succeed in being required yet not provide a specific needed feature.
+        def require_libddwaf
+          Datadog.logger.debug { "libddwaf platform: #{libddwaf_platform}" }
+
+          require 'libddwaf'
+
+          true
+        rescue LoadError => e
+          Datadog.logger.error do
+            'libddwaf failed to load,' \
+              "installed platform: #{libddwaf_platform} ruby platforms: #{ruby_platforms} error: #{e.inspect}"
+          end
+
+          false
+        end
+
+        def libddwaf_spec
+          Gem.loaded_specs['libddwaf']
+        end
+
+        def libddwaf_platform
+          libddwaf_spec ? libddwaf_spec.platform.to_s : 'unknown'
+        end
+
+        def ruby_platforms
+          Gem.platforms.map(&:to_s)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/rate_limiter.rb

@@ -0,0 +1,60 @@
+# typed: true
+
+module Datadog
+  module AppSec
+    # Simple per-thread rate limiter
+    # Since AppSec marks sampling to keep on a security event, this limits the flood of egress traces involving AppSec
+    class RateLimiter
+      def initialize(rate)
+        @rate = rate
+        @timestamps = []
+      end
+
+      def limit
+        now = Time.now.to_f
+
+        loop do
+          oldest = @timestamps.first
+
+          break if oldest.nil? || now - oldest < 1
+
+          @timestamps.shift
+        end
+
+        @timestamps << now
+
+        if (count = @timestamps.count) <= @rate
+          yield
+        else
+          Datadog.logger.debug { "Rate limit hit: #{count}/#{@rate} AppSec traces/second" }
+        end
+      end
+
+      class << self
+        def limit(name, &block)
+          rate_limiter(name).limit(&block)
+        end
+
+        # reset a rate limiter: used for testing
+        def reset!(name)
+          Thread.current[:datadog_security_trace_rate_limiter] = nil
+        end
+
+        protected
+
+        def rate_limiter(name)
+          case name
+          when :traces
+            Thread.current[:datadog_security_trace_rate_limiter] ||= RateLimiter.new(trace_rate_limit)
+          else
+            raise "unsupported rate limiter: #{name.inspect}"
+          end
+        end
+
+        def trace_rate_limit
+          Datadog::AppSec.settings.trace_rate_limit
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/reactive/address_hash.rb

@@ -0,0 +1,18 @@
+# typed: true
+
+module Datadog
+  module AppSec
+    module Reactive
+      # Address for Reactive Engine
+      class AddressHash < Hash
+        def addresses
+          keys.flatten
+        end
+
+        def with(address)
+          keys.select { |k| k.include?(address) }
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/reactive/engine.rb

@@ -0,0 +1,44 @@
+# typed: true
+
+require_relative 'address_hash'
+require_relative 'subscriber'
+
+module Datadog
+  module AppSec
+    module Reactive
+      # Reactive Engine
+      class Engine
+        def initialize
+          @data = {}
+          @subscribers = AddressHash.new { |h, k| h[k] = [] } # TODO: move to AddressHash initializer
+          @children = []
+        end
+
+        def subscribe(*addresses, &block)
+          @subscribers[addresses.freeze] << Subscriber.new(&block).freeze # TODO: move freeze to Subscriber
+        end
+
+        def publish(address, data)
+          # check if someone has address subscribed
+          if @subscribers.addresses.include?(address)
+
+            # someone will be interested, set data
+            @data[address] = data
+
+            # find candidates i.e address groups that contain the just posted address
+            @subscribers.with(address).each do |addresses|
+              # find targets to the address group containing the posted address
+              subscribers = @subscribers[addresses]
+
+              # is all data for the targets available?
+              if (addresses - @data.keys).empty?
+                hash = addresses.each_with_object({}) { |a, h| h[a] = @data[a] }
+                subscribers.each { |s| s.call(*hash.values) }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/reactive/operation.rb

@@ -0,0 +1,51 @@
+# typed: true
+
+require_relative 'engine'
+
+module Datadog
+  module AppSec
+    module Reactive
+      # Reactive Engine nested operation tracking
+      class Operation
+        attr_reader :reactive,
+          :parent,
+          :name
+
+        def initialize(name, parent = nil, reactive_engine = nil)
+          Datadog.logger.debug { "operation: #{name} initialize" }
+          @name = name
+          @parent = parent
+          @reactive = reactive_engine || parent && parent.reactive || Reactive::Engine.new
+
+          # TODO: concurrent store
+          # TODO: constant
+          Thread.current[:datadog_security_active_operation] = self
+
+          yield self if block_given?
+        ensure
+          finalize
+        end
+
+        # TODO: use structs instead of an arg splat
+        def subscribe(*addresses, &block)
+          reactive.subscribe(*addresses, &block)
+        end
+
+        def publish(address, data)
+          reactive.publish(address, data)
+        end
+
+        def finalize
+          Datadog.logger.debug { "operation: #{name} finalize" }
+          Thread.current[:datadog_security_active_operation] = parent
+        end
+
+        class << self
+          def active
+            Thread.current[:datadog_security_active_operation]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/reactive/subscriber.rb

@@ -0,0 +1,18 @@
+# typed: true
+
+module Datadog
+  module AppSec
+    module Reactive
+      # Reactive Engine subscriber
+      class Subscriber
+        def initialize(&block)
+          @block = block
+        end
+
+        def call(*args)
+          @block.call(*args)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/response.rb

@@ -0,0 +1,54 @@
+# typed: false
+
+require_relative 'assets'
+
+module Datadog
+  module AppSec
+    # AppSec response
+    class Response
+      attr_reader :status, :headers, :body
+
+      def initialize(status:, headers: {}, body: [])
+        @status = status
+        @headers = headers
+        @body = body
+      end
+
+      def to_rack
+        [status, headers, body]
+      end
+
+      def to_sinatra_response
+        ::Sinatra::Response.new(body, status, headers)
+      end
+
+      def to_action_dispatch_response
+        ::ActionDispatch::Response.new(status, headers, body)
+      end
+
+      class << self
+        def negotiate(env)
+          Response.new(
+            status: 403,
+            headers: { 'Content-Type' => 'text/html' },
+            body: [Datadog::AppSec::Assets.blocked(format: format(env))]
+          )
+        end
+
+        private
+
+        def format(env)
+          format = env['HTTP_ACCEPT'] && env['HTTP_ACCEPT'].split(',').any? do |accept|
+            if accept.start_with?('text/html')
+              break :html
+            elsif accept.start_with?('application/json')
+              break :json
+            end
+          end
+
+          format || :text
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec.rb

@@ -0,0 +1,23 @@
+# typed: false
+
+require_relative 'appsec/configuration'
+require_relative 'appsec/extensions'
+
+module Datadog
+  # Namespace for Datadog AppSec instrumentation
+  module AppSec
+    include Configuration
+
+    def self.writer
+      @writer ||= Writer.new
+    end
+
+    # Expose AppSec to global shared objects
+    Extensions.activate!
+  end
+end
+
+# Integrations
+require_relative 'appsec/contrib/rack/integration'
+require_relative 'appsec/contrib/sinatra/integration'
+require_relative 'appsec/contrib/rails/integration'

data/lib/datadog/ci/configuration/components.rb

@@ -1,5 +1,6 @@
 # typed: true
-require 'datadog/ci/context_flush'
+
+require_relative '../flush'
 
 module Datadog
   module CI
@@ -8,22 +9,22 @@ module Datadog
       module Components
         def initialize(settings)
           # Activate CI mode if enabled
-          activate_ci_mode!(settings) if settings.ci_mode.enabled
+          activate_ci!(settings) if settings.ci.enabled
 
           # Initialize normally
           super
         end
 
-        def activate_ci_mode!(settings)
+        def activate_ci!(settings)
           # Activate underlying tracing test mode
-          settings.test_mode.enabled = true
+          settings.tracing.test_mode.enabled = true
 
-          # Choose user defined ContextFlush or default to CI ContextFlush
-          settings.test_mode.context_flush = settings.ci_mode.context_flush \
-                                             || Datadog::CI::ContextFlush::Finished.new
+          # Choose user defined TraceFlush or default to CI TraceFlush
+          settings.tracing.test_mode.trace_flush = settings.ci.trace_flush \
+                                             || CI::Flush::Finished.new
 
           # Pass through any other options
-          settings.test_mode.writer_options = settings.ci_mode.writer_options
+          settings.tracing.test_mode.writer_options = settings.ci.writer_options
         end
       end
     end

data/lib/datadog/ci/configuration/settings.rb

@@ -1,5 +1,6 @@
 # typed: false
-require 'datadog/ci/ext/settings'
+
+require_relative '../ext/settings'
 
 module Datadog
   module CI
@@ -13,13 +14,28 @@ module Datadog
 
         def self.add_settings!(base)
           base.class_eval do
-            settings :ci_mode do
+            settings :ci do
               option :enabled do |o|
-                o.default { env_to_bool(Datadog::CI::Ext::Settings::ENV_MODE_ENABLED, false) }
+                o.default { env_to_bool(CI::Ext::Settings::ENV_MODE_ENABLED, false) }
                 o.lazy
               end
 
-              option :context_flush do |o|
+              # DEV: Alias to Datadog::Tracing::Contrib::Extensions::Configuration::Settings#instrument.
+              # DEV: Should be removed when CI implement its own `c.ci.instrument`.
+              define_method(:instrument) do |integration_name, options = {}, &block|
+                Datadog.configuration.tracing.instrument(integration_name, options, &block)
+              end
+
+              # DEV: Alias to Datadog::Tracing::Contrib::Extensions::Configuration::Settings#instrument.
+              # DEV: Should be removed when CI implement its own `c.ci[]`.
+              define_method(:[]) do |integration_name, key = :default|
+                Datadog.configuration.tracing[integration_name, key]
+              end
+
+              # TODO: Deprecate in the next major version, as `instrument` better describes this method's purpose
+              alias_method :use, :instrument
+
+              option :trace_flush do |o|
                 o.default { nil }
                 o.lazy
               end

data/lib/datadog/ci/contrib/cucumber/configuration/settings.rb

@@ -1,6 +1,7 @@
 # typed: false
-require 'ddtrace/contrib/configuration/settings'
-require 'datadog/ci/contrib/cucumber/ext'
+
+require_relative '../../../../tracing/contrib/configuration/settings'
+require_relative '../ext'
 
 module Datadog
   module CI
@@ -8,7 +9,8 @@ module Datadog
       module Cucumber
         module Configuration
           # Custom settings for the Cucumber integration
-          class Settings < Datadog::Contrib::Configuration::Settings
+          # TODO: mark as `@public_api` when GA
+          class Settings < Datadog::Tracing::Contrib::Configuration::Settings
             option :enabled do |o|
               o.default { env_to_bool(Ext::ENV_ENABLED, true) }
               o.lazy

data/lib/datadog/ci/contrib/cucumber/ext.rb

@@ -1,9 +1,11 @@
 # typed: true
+
 module Datadog
   module CI
     module Contrib
       module Cucumber
         # Cucumber integration constants
+        # TODO: mark as `@public_api` when GA, to protect from resource and tag name changes.
         module Ext
           APP = 'cucumber'.freeze
           ENV_ENABLED = 'DD_TRACE_CUCUMBER_ENABLED'.freeze

data/lib/datadog/ci/contrib/cucumber/formatter.rb

@@ -1,9 +1,10 @@
 # typed: true
-require 'datadog/ci/test'
-require 'datadog/ci/ext/app_types'
-require 'datadog/ci/ext/environment'
-require 'datadog/ci/ext/test'
-require 'datadog/ci/contrib/cucumber/ext'
+
+require_relative '../../test'
+require_relative '../../ext/app_types'
+require_relative '../../ext/environment'
+require_relative '../../ext/test'
+require_relative 'ext'
 
 module Datadog
   module CI
@@ -30,16 +31,14 @@ module Datadog
 
           def on_test_case_started(event)
             @current_feature_span = CI::Test.trace(
-              tracer,
               configuration[:operation_name],
               {
                 span_options: {
-                  app: Ext::APP,
                   resource: event.test_case.name,
                   service: configuration[:service_name]
                 },
                 framework: Ext::FRAMEWORK,
-                framework_version: Datadog::CI::Contrib::Cucumber::Integration.version.to_s,
+                framework_version: CI::Contrib::Cucumber::Integration.version.to_s,
                 test_name: event.test_case.name,
                 test_suite: event.test_case.location.file,
                 test_type: Ext::TEST_TYPE
@@ -66,7 +65,7 @@ module Datadog
               resource: event.test_step.to_s,
               span_type: Ext::STEP_SPAN_TYPE
             }
-            @current_step_span = tracer.trace(Ext::STEP_SPAN_TYPE, trace_options)
+            @current_step_span = Tracing.trace(Ext::STEP_SPAN_TYPE, **trace_options)
           end
 
           def on_test_step_finished(event)
@@ -86,11 +85,7 @@ module Datadog
           private
 
           def configuration
-            Datadog.configuration[:cucumber]
-          end
-
-          def tracer
-            configuration[:tracer]
+            Datadog.configuration.ci[:cucumber]
           end
         end
       end

data/lib/datadog/ci/contrib/cucumber/instrumentation.rb

@@ -1,5 +1,6 @@
 # typed: true
-require 'datadog/ci/contrib/cucumber/formatter'
+
+require_relative 'formatter'
 
 module Datadog
   module CI
@@ -16,7 +17,7 @@ module Datadog
             attr_reader :datadog_formatter
 
             def formatters
-              @datadog_formatter ||= Datadog::CI::Contrib::Cucumber::Formatter.new(@configuration)
+              @datadog_formatter ||= CI::Contrib::Cucumber::Formatter.new(@configuration)
               [@datadog_formatter] + super
             end
           end

data/lib/datadog/ci/contrib/cucumber/integration.rb

@@ -1,8 +1,9 @@
 # typed: false
-require 'ddtrace/contrib/integration'
 
-require 'datadog/ci/contrib/cucumber/configuration/settings'
-require 'datadog/ci/contrib/cucumber/patcher'
+require_relative '../../../tracing/contrib/integration'
+
+require_relative 'configuration/settings'
+require_relative 'patcher'
 
 module Datadog
   module CI
@@ -10,7 +11,7 @@ module Datadog
       module Cucumber
         # Description of Cucumber integration
         class Integration
-          include Datadog::Contrib::Integration
+          include Datadog::Tracing::Contrib::Integration
 
           MINIMUM_VERSION = Gem::Version.new('3.0.0')
 
@@ -34,7 +35,7 @@ module Datadog
             false
           end
 
-          def default_configuration
+          def new_configuration
             Configuration::Settings.new
           end