datadog 2.7.1 → 2.18.0

data/lib/datadog/appsec/security_engine/result.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module SecurityEngine
+      # A namespace for value-objects representing the result of WAF check.
+      module Result
+        # A generic result without indication of its type.
+        class Base
+          attr_reader :events, :actions, :derivatives, :duration_ns, :duration_ext_ns
+
+          def initialize(events:, actions:, derivatives:, timeout:, duration_ns:, duration_ext_ns:)
+            @events = events
+            @actions = actions
+            @derivatives = derivatives
+
+            @timeout = timeout
+            @duration_ns = duration_ns
+            @duration_ext_ns = duration_ext_ns
+          end
+
+          def timeout?
+            !!@timeout
+          end
+
+          def match?
+            raise NotImplementedError
+          end
+        end
+
+        # A result that indicates a security rule match
+        class Match < Base
+          def match?
+            true
+          end
+        end
+
+        # A result that indicates a successful security rules check without a match
+        class Ok < Base
+          def match?
+            false
+          end
+        end
+
+        # A result that indicates an internal security library error
+        class Error
+          attr_reader :events, :actions, :derivatives, :duration_ns, :duration_ext_ns
+
+          def initialize(duration_ext_ns:)
+            @events = []
+            @actions = @derivatives = {}
+            @duration_ns = 0
+            @duration_ext_ns = duration_ext_ns
+          end
+
+          def timeout?
+            false
+          end
+
+          def match?
+            false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/security_engine/runner.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+require_relative 'result'
+
+module Datadog
+  module AppSec
+    module SecurityEngine
+      # A class that check input via security engine (WAF) and respond with result.
+      class Runner
+        SUCCESSFUL_EXECUTION_CODES = [:ok, :match].freeze
+
+        def initialize(waf_context)
+          @mutex = Mutex.new
+          @waf_context = waf_context
+
+          @debug_tag = "libddwaf:#{WAF::VERSION::STRING} method:ddwaf_run"
+        end
+
+        def run(persistent_data, ephemeral_data, timeout = WAF::LibDDWAF::DDWAF_RUN_TIMEOUT)
+          @mutex.lock
+
+          start_ns = Core::Utils::Time.get_time(:nanosecond)
+          persistent_data.reject! do |_, v|
+            next false if v.is_a?(TrueClass) || v.is_a?(FalseClass)
+
+            v.nil? || v.empty?
+          end
+
+          ephemeral_data.reject! do |_, v|
+            next false if v.is_a?(TrueClass) || v.is_a?(FalseClass)
+
+            v.nil? || v.empty?
+          end
+
+          result = try_run(persistent_data, ephemeral_data, timeout)
+          stop_ns = Core::Utils::Time.get_time(:nanosecond)
+
+          report_execution(result)
+
+          unless SUCCESSFUL_EXECUTION_CODES.include?(result.status)
+            return Result::Error.new(duration_ext_ns: stop_ns - start_ns)
+          end
+
+          klass = (result.status == :match) ? Result::Match : Result::Ok
+          klass.new(
+            events: result.events,
+            actions: result.actions,
+            derivatives: result.derivatives,
+            timeout: result.timeout,
+            duration_ns: result.total_runtime,
+            duration_ext_ns: (stop_ns - start_ns)
+          )
+        ensure
+          @mutex.unlock
+        end
+
+        def finalize!
+          @waf_context.finalize!
+        end
+
+        private
+
+        def try_run(persistent_data, ephemeral_data, timeout)
+          @waf_context.run(persistent_data, ephemeral_data, timeout)
+        rescue WAF::LibDDWAFError => e
+          Datadog.logger.debug { "#{@debug_tag} execution error: #{e} backtrace: #{e.backtrace&.first(3)}" }
+          AppSec.telemetry.report(e, description: 'libddwaf-rb internal low-level error')
+
+          WAF::Result.new(:err_internal, [], 0, false, [], [])
+        end
+
+        def report_execution(result)
+          Datadog.logger.debug { "#{@debug_tag} execution timed out: #{result.inspect}" } if result.timeout
+
+          if SUCCESSFUL_EXECUTION_CODES.include?(result.status)
+            Datadog.logger.debug { "#{@debug_tag} execution result: #{result.inspect}" }
+          else
+            message = "#{@debug_tag} execution error: #{result.status.inspect}"
+
+            Datadog.logger.debug { message }
+            AppSec.telemetry.error(message)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/security_engine.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    # A namespace for secutiry library we use to detect and prevent threats.
+    module SecurityEngine
+    end
+  end
+end

data/lib/datadog/appsec/security_event.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    # A class that represents a security event of any kind. It could be an event
+    # representing an attack or fingerprinting results as derivatives or an API
+    # security check with extracted schema.
+    class SecurityEvent
+      SCHEMA_KEY_PREFIX = '_dd.appsec.s.'
+      FINGERPRINT_KEY_PREFIX = '_dd.appsec.fp.'
+
+      attr_reader :waf_result, :trace, :span
+
+      def initialize(waf_result, trace:, span:)
+        @waf_result = waf_result
+        @trace = trace
+        @span = span
+      end
+
+      def attack?
+        return @is_attack if defined?(@is_attack)
+
+        @is_attack = @waf_result.is_a?(SecurityEngine::Result::Match)
+      end
+
+      def schema?
+        return @has_schema if defined?(@has_schema)
+
+        @has_schema = @waf_result.derivatives.any? { |name, _| name.start_with?(SCHEMA_KEY_PREFIX) }
+      end
+
+      def fingerprint?
+        return @has_fingerprint if defined?(@has_fingerprint)
+
+        @has_fingerprint = @waf_result.derivatives.any? { |name, _| name.start_with?(FINGERPRINT_KEY_PREFIX) }
+      end
+    end
+  end
+end

data/lib/datadog/appsec/utils.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require_relative 'utils/trace_operation'
-
 module Datadog
   module AppSec
     # Utilities for AppSec

data/lib/datadog/appsec.rb

@@ -2,7 +2,7 @@
 
 require_relative 'appsec/configuration'
 require_relative 'appsec/extensions'
-require_relative 'appsec/scope'
+require_relative 'appsec/context'
 require_relative 'appsec/ext'
 require_relative 'appsec/utils'
 
@@ -14,32 +14,38 @@ module Datadog
         Datadog.configuration.appsec.enabled
       end
 
-      def active_scope
-        Datadog::AppSec::Scope.active_scope
+      def rasp_enabled?
+        Datadog.configuration.appsec.rasp_enabled
       end
 
-      def processor
-        appsec_component = components.appsec
-
-        appsec_component.processor if appsec_component
+      def active_context
+        Datadog::AppSec::Context.active
       end
 
-      def reconfigure(ruleset:, actions:, telemetry:)
-        appsec_component = components.appsec
+      def telemetry
+        components.appsec&.telemetry
+      end
 
-        return unless appsec_component
+      def security_engine
+        components.appsec&.security_engine
+      end
 
-        appsec_component.reconfigure(ruleset: ruleset, actions: actions, telemetry: telemetry)
+      def reconfigure!
+        components.appsec&.reconfigure!
       end
 
       def reconfigure_lock(&block)
         appsec_component = components.appsec
-
         return unless appsec_component
 
         appsec_component.reconfigure_lock(&block)
       end
 
+      def perform_api_security_check?
+        Datadog.configuration.appsec.api_security.enabled &&
+          Datadog.configuration.appsec.api_security.sample_rate.sample?
+      end
+
       private
 
       def components
@@ -56,7 +62,11 @@ end
 require_relative 'appsec/contrib/rack/integration'
 require_relative 'appsec/contrib/sinatra/integration'
 require_relative 'appsec/contrib/rails/integration'
+require_relative 'appsec/contrib/active_record/integration'
 require_relative 'appsec/contrib/devise/integration'
 require_relative 'appsec/contrib/graphql/integration'
+require_relative 'appsec/contrib/faraday/integration'
+require_relative 'appsec/contrib/excon/integration'
+require_relative 'appsec/contrib/rest_client/integration'
 
 require_relative 'appsec/autoload'

data/lib/datadog/auto_instrument.rb

@@ -6,6 +6,9 @@
 require_relative '../datadog'
 require_relative 'tracing/contrib/auto_instrument'
 
+# DI is not loaded on Ruby 2.5 and JRuby
+Datadog::DI::Contrib.load_now_or_later if defined?(Datadog::DI::Contrib)
+
 Datadog::Profiling.start_if_enabled
 
 module Datadog

data/lib/datadog/core/buffer/random.rb

@@ -40,7 +40,23 @@ module Datadog
           add_all!(underflow) unless underflow.nil?
 
           # Iteratively replace items, to ensure pseudo-random replacement.
-          overflow.each { |item| replace!(item) } unless overflow.nil?
+          overflow&.each { |item| replace!(item) }
+        end
+
+        def unshift(*items)
+          # TODO The existing concat implementation does not always append
+          # to the end of the buffer - if the buffer is full, a random
+          # item is deleted and the new item is added in the position of
+          # removed item.
+          # Therefore, if we want to preserve the item order, concat
+          # would also need to be changed to maintain order.
+          # With the existing implementation, the idea is to not move
+          # existing items around, which is what sets unshift apart from
+          # concat to begin with.
+          #
+          # Since this method currently delegates to +concat+, it does not
+          # have a matching definition in the thread-safe worker.
+          concat(items)
         end
 
         # Stored items are returned and the local buffer is reset.
@@ -78,7 +94,7 @@ module Datadog
           underflow = nil
           overflow = nil
 
-          overflow_size = @max_size > 0 ? (@items.length + items.length) - @max_size : 0
+          overflow_size = (@max_size > 0) ? (@items.length + items.length) - @max_size : 0
 
           if overflow_size > 0
             # Items will overflow

data/lib/datadog/core/configuration/agent_settings.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require_relative 'ext'
+
+module Datadog
+  module Core
+    module Configuration
+      # Immutable container for the resulting settings
+      class AgentSettings
+        # IPv6 regular expression from
+        # https://stackoverflow.com/questions/53497/regular-expression-that-matches-valid-ipv6-addresses
+        # Does not match IPv4 addresses.
+        IPV6_REGEXP = /\A(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\z)/.freeze # rubocop:disable Layout/LineLength
+
+        attr_reader :adapter, :ssl, :hostname, :port, :uds_path, :timeout_seconds
+
+        def initialize(adapter: nil, ssl: nil, hostname: nil, port: nil, uds_path: nil, timeout_seconds: nil)
+          @adapter = adapter
+          @ssl = ssl
+          @hostname = hostname
+          @port = port
+          @uds_path = uds_path
+          @timeout_seconds = timeout_seconds
+          freeze
+        end
+
+        def url
+          case adapter
+          when Datadog::Core::Configuration::Ext::Agent::HTTP::ADAPTER
+            hostname = self.hostname
+            hostname = "[#{hostname}]" if IPV6_REGEXP.match?(hostname)
+            "#{ssl ? "https" : "http"}://#{hostname}:#{port}/"
+          when Datadog::Core::Configuration::Ext::Agent::UnixSocket::ADAPTER
+            "unix://#{uds_path}"
+          else
+            raise ArgumentError, "Unexpected adapter: #{adapter}"
+          end
+        end
+
+        def ==(other)
+          self.class == other.class &&
+            adapter == other.adapter &&
+            ssl == other.ssl &&
+            hostname == other.hostname &&
+            port == other.port &&
+            uds_path == other.uds_path &&
+            timeout_seconds == other.timeout_seconds
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/configuration/agent_settings_resolver.rb

@@ -4,6 +4,7 @@ require 'uri'
 
 require_relative 'settings'
 require_relative 'ext'
+require_relative 'agent_settings'
 require_relative '../transport/ext'
 
 module Datadog
@@ -19,21 +20,6 @@ module Datadog
       # Whenever there is a conflict (different configurations are provided in different orders), it MUST warn the users
       # about it and pick a value based on the following priority: code > environment variable > defaults.
       class AgentSettingsResolver
-        AgentSettings = Struct.new(
-          :adapter,
-          :ssl,
-          :hostname,
-          :port,
-          :uds_path,
-          :timeout_seconds,
-          keyword_init: true
-        ) do
-          def initialize(*)
-            super
-            freeze
-          end
-        end
-
         def self.call(settings, logger: Datadog.logger)
           new(settings, logger: logger).send(:call)
         end
@@ -130,7 +116,7 @@ module Datadog
               value: settings.agent.timeout_seconds,
             ),
             try_parsing_as_integer(
-              friendly_name: "#{Datadog::Core::Configuration::Ext::Agent::ENV_DEFAULT_TIMEOUT_SECONDS} "\
+              friendly_name: "#{Datadog::Core::Configuration::Ext::Agent::ENV_DEFAULT_TIMEOUT_SECONDS} " \
                 'environment variable',
               value: ENV[Datadog::Core::Configuration::Ext::Agent::ENV_DEFAULT_TIMEOUT_SECONDS],
             )
@@ -310,13 +296,13 @@ module Datadog
           log_warning(
             'Configuration mismatch: values differ between ' \
             "#{detected_configurations_in_priority_order
-              .map { |config| "#{config.friendly_name} (#{config.value.inspect})" }.join(' and ')}" \
+              .map { |config| "#{config.friendly_name} (#{config.value.inspect})" }.join(" and ")}" \
             ". Using #{detected_configurations_in_priority_order.first.value.inspect} and ignoring other configuration."
           )
         end
 
         def log_warning(message)
-          logger.warn(message) if logger
+          logger&.warn(message)
         end
 
         def http_scheme?(uri)

data/lib/datadog/core/configuration/agentless_settings_resolver.rb

@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+# rubocop:disable Style/*
+
+require 'uri'
+
+require_relative 'agent_settings_resolver'
+
+module Datadog
+  module Core
+    module Configuration
+      # Agent settings resolver for agentless operations (currently, telemetry
+      # in agentless mode).
+      #
+      # The terminology gets a little confusing here, but transports communicate
+      # with servers which are - for most components in the tracer - the
+      # (local) agent. Hence, "agent settings" to refer to where the server
+      # is located. Telemetry supports sending to the local agent but also
+      # implements agentless mode where it sends directly to Datadog intake
+      # endpoints. The agentless mode is configured using different settings,
+      # and this class produces AgentSettings instances when in agentless mode.
+      #
+      # Agentless settings resolver uses the following configuration sources:
+      #
+      # 1. url_override constructor parameter, if provided
+      # 2. Built-in default host/port/TLS settings for the backend
+      #    intake endpoint
+      #
+      # The agentless resolver does NOT use agent settings (since it is
+      # for agentless operation), specifically it ignores:
+      #
+      # - c.agent.host
+      # - DD_AGENT_HOST
+      # - c.agent.port
+      # - DD_AGENT_PORT
+      #
+      # However, agentless resolver does respect the timeout specified via
+      # c.agent.timeout_seconds or DD_TRACE_AGENT_TIMEOUT_SECONDS.
+      class AgentlessSettingsResolver < AgentSettingsResolver
+        # To avoid coupling this class to telemetry, the URL override is
+        # taken here as a parameter instead of being read out of
+        # c.telemetry.agentless_url_override. For the same reason, the
+        # +url_override_source+ parameter should be set to the string
+        # "c.telemetry.agentless_url_override".
+        def self.call(settings, host_prefix:, url_override: nil, url_override_source: nil, logger: Datadog.logger)
+          new(
+            settings,
+            host_prefix: host_prefix,
+            url_override: url_override,
+            url_override_source: url_override_source,
+            logger: logger
+          ).send(:call)
+        end
+
+        private
+
+        attr_reader \
+          :host_prefix,
+          :url_override,
+          :url_override_source
+
+        def initialize(settings, host_prefix:, url_override: nil, url_override_source: nil, logger: Datadog.logger)
+          if url_override && url_override_source.nil?
+            raise ArgumentError, 'url_override_source must be provided when url_override is provided'
+          end
+
+          super(settings, logger: logger)
+
+          @host_prefix = host_prefix
+          @url_override = url_override
+          @url_override_source = url_override_source
+        end
+
+        def hostname
+          if should_use_uds?
+            nil
+          else
+            configured_hostname || "#{host_prefix}.#{settings.site}"
+          end
+        end
+
+        def configured_hostname
+          return @configured_hostname if defined?(@configured_hostname)
+
+          if should_use_uds?
+            nil
+          else
+            @configured_hostname = (parsed_url.hostname if parsed_url)
+          end
+        end
+
+        def configured_port
+          return @configured_port if defined?(@configured_port)
+
+          @configured_port = (parsed_url.port if parsed_url)
+        end
+
+        # Note that this method should always return true or false
+        def ssl?
+          if configured_hostname
+            configured_ssl || false
+          else
+            if should_use_uds?
+              false
+            else
+              # If no hostname is specified, we are communicating with the
+              # default Datadog intake, which uses TLS.
+              true
+            end
+          end
+        end
+
+        # Note that this method can return nil
+        def configured_ssl
+          return @configured_ssl if defined?(@configured_ssl)
+
+          @configured_ssl = (parsed_url_ssl? if parsed_url)
+        end
+
+        def port
+          if configured_port
+            configured_port
+          else
+            if should_use_uds?
+              nil
+            else
+              # If no hostname is specified, we are communicating with the
+              # default Datadog intake, which exists on port 443.
+              443
+            end
+          end
+        end
+
+        def mixed_http_and_uds
+          false
+        end
+
+        def configured_uds_path
+          return @configured_uds_path if defined?(@configured_uds_path)
+
+          parsed_url_uds_path
+        end
+
+        def can_use_uds?
+          # While in theory agentless transport could communicate via UDS,
+          # in practice "agentless" means we are communicating with Datadog
+          # infrastructure which is always remote.
+          # Permit UDS for proxy usage?
+          !configured_uds_path.nil?
+        end
+
+        def parsed_url
+          return @parsed_url if defined?(@parsed_url)
+
+          @parsed_url =
+            if @url_override
+              parsed = URI.parse(@url_override)
+
+              # Agentless URL should never refer to a UDS?
+              if http_scheme?(parsed) || unix_scheme?(parsed)
+                parsed
+              else
+                log_warning(
+                  "Invalid URI scheme '#{parsed.scheme}' for #{url_override_source}. " \
+                  "Ignoring the contents of #{url_override_source}."
+                )
+                nil
+              end
+            end
+        end
+      end
+    end
+  end
+end
+
+# rubocop:enable Style/*