datadog 2.15.0 → 2.16.0

data/lib/datadog/appsec/monitor/gateway/watcher.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative '../../event'
+require_relative '../../security_event'
 require_relative '../../instrumentation/gateway'
 
 module Datadog
@@ -8,18 +10,24 @@ module Datadog
       module Gateway
         # Watcher for Apssec internal events
         module Watcher
+          ARBITRARY_VALUE = 'invalid'
+          EVENT_LOGIN_SUCCESS = 'users.login.success'
+          EVENT_LOGIN_FAILURE = 'users.login.failure'
+          WATCHED_LOGIN_EVENTS = [EVENT_LOGIN_SUCCESS, EVENT_LOGIN_FAILURE].freeze
+
           class << self
             def watch
               gateway = Instrumentation.gateway
 
               watch_user_id(gateway)
+              watch_user_login(gateway)
             end
 
             def watch_user_id(gateway = Instrumentation.gateway)
               gateway.watch('identity.set_user', :appsec) do |stack, user|
-                context = Datadog::AppSec.active_context
+                context = AppSec.active_context
 
-                if user.id.nil? && user.login.nil?
+                if user.id.nil? && user.login.nil? && user.session_id.nil?
                   Datadog.logger.debug { 'AppSec: skipping WAF check because no user information was provided' }
                   next stack.call(user)
                 end
@@ -27,24 +35,46 @@ module Datadog
                 persistent_data = {}
                 persistent_data['usr.id'] = user.id if user.id
                 persistent_data['usr.login'] = user.login if user.login
+                persistent_data['usr.session_id'] = user.session_id if user.session_id
 
                 result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
 
+                if result.match? || result.derivatives.any?
+                  context.events.push(
+                    AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
+                  )
+                end
+
                 if result.match?
-                  Datadog::AppSec::Event.tag_and_keep!(context, result)
+                  AppSec::Event.tag_and_keep!(context, result)
+                  AppSec::ActionsHandler.handle(result.actions)
+                end
+
+                stack.call(user)
+              end
+            end
 
-                  context.events << {
-                    waf_result: result,
-                    trace: context.trace,
-                    span: context.span,
-                    user: user,
-                    actions: result.actions
-                  }
+            def watch_user_login(gateway = Instrumentation.gateway)
+              gateway.watch('appsec.events.user_lifecycle', :appsec) do |stack, kind|
+                context = AppSec.active_context
 
-                  Datadog::AppSec::ActionsHandler.handle(result.actions)
+                next stack.call(kind) unless WATCHED_LOGIN_EVENTS.include?(kind)
+
+                persistent_data = {"server.business_logic.#{kind}" => ARBITRARY_VALUE}
+                result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
+
+                if result.match? || result.derivatives.any?
+                  context.events.push(
+                    AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
+                  )
                 end
 
-                stack.call(user)
+                if result.match?
+                  AppSec::Event.tag_and_keep!(context, result)
+                  AppSec::ActionsHandler.handle(result.actions)
+                end
+
+                stack.call(kind)
               end
             end
           end

data/lib/datadog/appsec/processor/rule_loader.rb

@@ -10,35 +10,33 @@ module Datadog
       module RuleLoader
         class << self
           def load_rules(ruleset:, telemetry:)
-            begin
-              case ruleset
-              when :recommended, :strict
-                JSON.parse(Datadog::AppSec::Assets.waf_rules(ruleset))
-              when :risky
-                Datadog.logger.warn(
-                  'The :risky Application Security Management ruleset has been deprecated and no longer available.'\
-                  'The `:recommended` ruleset will be used instead.'\
-                  'Please remove the `appsec.ruleset = :risky` setting from your Datadog.configure block.'
-                )
-                JSON.parse(Datadog::AppSec::Assets.waf_rules(:recommended))
-              when String
-                JSON.parse(File.read(File.expand_path(ruleset)))
-              when File, StringIO
-                JSON.parse(ruleset.read || '').tap { ruleset.rewind }
-              when Hash
-                ruleset
-              else
-                raise ArgumentError, "unsupported value for ruleset setting: #{ruleset.inspect}"
-              end
-            rescue StandardError => e
-              Datadog.logger.error do
-                "libddwaf ruleset failed to load, ruleset: #{ruleset.inspect} error: #{e.inspect}"
-              end
+            case ruleset
+            when :recommended, :strict
+              JSON.parse(Datadog::AppSec::Assets.waf_rules(ruleset))
+            when :risky
+              Datadog.logger.warn(
+                'The :risky Application Security Management ruleset has been deprecated and no longer available.' \
+                'The `:recommended` ruleset will be used instead.' \
+                'Please remove the `appsec.ruleset = :risky` setting from your Datadog.configure block.'
+              )
+              JSON.parse(Datadog::AppSec::Assets.waf_rules(:recommended))
+            when String
+              JSON.parse(File.read(File.expand_path(ruleset)))
+            when File, StringIO
+              JSON.parse(ruleset.read || '').tap { ruleset.rewind }
+            when Hash
+              ruleset
+            else
+              raise ArgumentError, "unsupported value for ruleset setting: #{ruleset.inspect}"
+            end
+          rescue => e
+            Datadog.logger.error do
+              "libddwaf ruleset failed to load, ruleset: #{ruleset.inspect} error: #{e.inspect}"
+            end
 
-              telemetry.report(e, description: 'libddwaf ruleset failed to load')
+            telemetry.report(e, description: 'libddwaf ruleset failed to load')
 
-              nil
-            end
+            nil
           end
 
           def load_data(ip_denylist: [], user_id_denylist: [])
@@ -62,7 +60,7 @@ module Datadog
             {
               'id' => id,
               'type' => 'data_with_expiration',
-              'data' => denylist.map { |v| { 'value' => v.to_s, 'expiration' => 2**63 } }
+              'data' => denylist.map { |v| {'value' => v.to_s, 'expiration' => 2**63} }
             }
           end
 

data/lib/datadog/appsec/processor/rule_merger.rb

@@ -11,8 +11,8 @@ module Datadog
         # RuleVersionMismatchError
         class RuleVersionMismatchError < StandardError
           def initialize(version1, version2)
-            msg = 'Merging rule files with different version could lead to unkown behaviour. '\
-              "We have receieve two rule files with versions: #{version1}, #{version2}. "\
+            msg = 'Merging rule files with different version could lead to unkown behaviour. ' \
+              "We have receieve two rule files with versions: #{version1}, #{version2}. " \
               'Please validate the configuration is correct and try again.'
             super(msg)
           end
@@ -27,7 +27,7 @@ module Datadog
           )
             processors ||= begin
               default_waf_processors
-            rescue StandardError => e
+            rescue => e
               Datadog.logger.error("libddwaf rulemerger failed to parse default waf processors. Error: #{e.inspect}")
               telemetry.report(
                 e,
@@ -38,7 +38,7 @@ module Datadog
 
             scanners ||= begin
               default_waf_scanners
-            rescue StandardError => e
+            rescue => e
               Datadog.logger.error("libddwaf rulemerger failed to parse default waf scanners. Error: #{e.inspect}")
               telemetry.report(
                 e,
@@ -146,7 +146,7 @@ module Datadog
             end
 
             result.each_with_object([]) do |entry, acc|
-              value = { 'value' => entry[0] }
+              value = {'value' => entry[0]}
               value['expiration'] = entry[1] if entry[1]
 
               acc << value

data/lib/datadog/appsec/processor.rb

@@ -82,7 +82,7 @@ module Datadog
         @diagnostics = e.diagnostics if e.diagnostics
 
         false
-      rescue StandardError => e
+      rescue => e
         Datadog.logger.error do
           "libddwaf failed to initialize, error: #{e.inspect}"
         end

data/lib/datadog/appsec/remote.rb

@@ -9,22 +9,23 @@ module Datadog
     # Remote
     module Remote
       class ReadError < StandardError; end
+
       class NoRulesError < StandardError; end
 
       class << self
-        CAP_ASM_RESERVED_1                = 1 << 0   # RESERVED
-        CAP_ASM_ACTIVATION                = 1 << 1   # Remote activation via ASM_FEATURES product
-        CAP_ASM_IP_BLOCKING               = 1 << 2   # accept IP blocking data from ASM_DATA product
-        CAP_ASM_DD_RULES                  = 1 << 3   # read ASM rules from ASM_DD product
-        CAP_ASM_EXCLUSIONS                = 1 << 4   # exclusion filters (passlist) via ASM product
-        CAP_ASM_REQUEST_BLOCKING          = 1 << 5   # can block on request info
-        CAP_ASM_RESPONSE_BLOCKING         = 1 << 6   # can block on response info
-        CAP_ASM_USER_BLOCKING             = 1 << 7   # accept user blocking data from ASM_DATA product
-        CAP_ASM_CUSTOM_RULES              = 1 << 8   # accept custom rules
-        CAP_ASM_CUSTOM_BLOCKING_RESPONSE  = 1 << 9   # supports custom http code or redirect sa blocking response
-        CAP_ASM_TRUSTED_IPS               = 1 << 10  # supports trusted ip
-        CAP_ASM_RASP_SSRF                 = 1 << 23  # support for server-side request forgery exploit prevention rules
-        CAP_ASM_RASP_SQLI                 = 1 << 21  # support for SQL injection exploit prevention rules
+        CAP_ASM_RESERVED_1 = 1 << 0   # RESERVED
+        CAP_ASM_ACTIVATION = 1 << 1   # Remote activation via ASM_FEATURES product
+        CAP_ASM_IP_BLOCKING = 1 << 2   # accept IP blocking data from ASM_DATA product
+        CAP_ASM_DD_RULES = 1 << 3   # read ASM rules from ASM_DD product
+        CAP_ASM_EXCLUSIONS = 1 << 4   # exclusion filters (passlist) via ASM product
+        CAP_ASM_REQUEST_BLOCKING = 1 << 5   # can block on request info
+        CAP_ASM_RESPONSE_BLOCKING = 1 << 6   # can block on response info
+        CAP_ASM_USER_BLOCKING = 1 << 7   # accept user blocking data from ASM_DATA product
+        CAP_ASM_CUSTOM_RULES = 1 << 8   # accept custom rules
+        CAP_ASM_CUSTOM_BLOCKING_RESPONSE = 1 << 9   # supports custom http code or redirect sa blocking response
+        CAP_ASM_TRUSTED_IPS = 1 << 10  # supports trusted ip
+        CAP_ASM_RASP_SSRF = 1 << 23  # support for server-side request forgery exploit prevention rules
+        CAP_ASM_RASP_SQLI = 1 << 21  # support for SQL injection exploit prevention rules
 
         # TODO: we need to dynamically add CAP_ASM_ACTIVATION once we support it
         ASM_CAPABILITIES = [

data/lib/datadog/appsec/response.rb

@@ -30,13 +30,13 @@ module Datadog
 
         def block_response(interrupt_params, http_accept_header)
           content_type = case interrupt_params['type']
-                         when nil, 'auto' then content_type(http_accept_header)
-                         else FORMAT_TO_CONTENT_TYPE.fetch(interrupt_params['type'], DEFAULT_CONTENT_TYPE)
-                         end
+          when nil, 'auto' then content_type(http_accept_header)
+          else FORMAT_TO_CONTENT_TYPE.fetch(interrupt_params['type'], DEFAULT_CONTENT_TYPE)
+          end
 
           Response.new(
             status: interrupt_params['status_code']&.to_i || 403,
-            headers: { 'Content-Type' => content_type },
+            headers: {'Content-Type' => content_type},
             body: [content(content_type)],
           )
         end
@@ -45,8 +45,8 @@ module Datadog
           status_code = interrupt_params['status_code'].to_i
 
           Response.new(
-            status: (status_code >= 300 && status_code < 400 ? status_code : 303),
-            headers: { 'Location' => interrupt_params.fetch('location') },
+            status: ((status_code >= 300 && status_code < 400) ? status_code : 303),
+            headers: {'Location' => interrupt_params.fetch('location')},
             body: [],
           )
         end

data/lib/datadog/appsec/security_engine/runner.rb

@@ -42,7 +42,7 @@ module Datadog
             return Result::Error.new(duration_ext_ns: stop_ns - start_ns)
           end
 
-          klass = result.status == :match ? Result::Match : Result::Ok
+          klass = (result.status == :match) ? Result::Match : Result::Ok
           klass.new(
             events: result.events,
             actions: result.actions,

data/lib/datadog/appsec/security_event.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    # A class that represents a security event of any kind. It could be an event
+    # representing an attack or fingerprinting results as derivatives or an API
+    # security check with extracted schema.
+    class SecurityEvent
+      SCHEMA_KEY_PREFIX = '_dd.appsec.s.'
+      FINGERPRINT_KEY_PREFIX = '_dd.appsec.fp.'
+
+      attr_reader :waf_result, :trace, :span
+
+      def initialize(waf_result, trace:, span:)
+        @waf_result = waf_result
+        @trace = trace
+        @span = span
+      end
+
+      def attack?
+        return @is_attack if defined?(@is_attack)
+
+        @is_attack = @waf_result.is_a?(SecurityEngine::Result::Match)
+      end
+
+      def schema?
+        return @has_schema if defined?(@has_schema)
+
+        @has_schema = @waf_result.derivatives.any? { |name, _| name.start_with?(SCHEMA_KEY_PREFIX) }
+      end
+
+      def fingerprint?
+        return @has_fingerprint if defined?(@has_fingerprint)
+
+        @has_fingerprint = @waf_result.derivatives.any? { |name, _| name.start_with?(FINGERPRINT_KEY_PREFIX) }
+      end
+    end
+  end
+end

data/lib/datadog/appsec.rb

@@ -44,7 +44,7 @@ module Datadog
         appsec_component.reconfigure_lock(&block)
       end
 
-      def api_security_enabled?
+      def perform_api_security_check?
         Datadog.configuration.appsec.api_security.enabled &&
           Datadog.configuration.appsec.api_security.sample_rate.sample?
       end

data/lib/datadog/core/configuration/agentless_settings_resolver.rb

@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+# rubocop:disable Style/*
+
+require 'uri'
+
+require_relative 'agent_settings_resolver'
+
+module Datadog
+  module Core
+    module Configuration
+      # Agent settings resolver for agentless operations (currently, telemetry
+      # in agentless mode).
+      #
+      # The terminology gets a little confusing here, but transports communicate
+      # with servers which are - for most components in the tracer - the
+      # (local) agent. Hence, "agent settings" to refer to where the server
+      # is located. Telemetry supports sending to the local agent but also
+      # implements agentless mode where it sends directly to Datadog intake
+      # endpoints. The agentless mode is configured using different settings,
+      # and this class produces AgentSettings instances when in agentless mode.
+      #
+      # Agentless settings resolver uses the following configuration sources:
+      #
+      # 1. url_override constructor parameter, if provided
+      # 2. Built-in default host/port/TLS settings for the backend
+      #    intake endpoint
+      #
+      # The agentless resolver does NOT use agent settings (since it is
+      # for agentless operation), specifically it ignores:
+      #
+      # - c.agent.host
+      # - DD_AGENT_HOST
+      # - c.agent.port
+      # - DD_AGENT_PORT
+      #
+      # However, agentless resolver does respect the timeout specified via
+      # c.agent.timeout_seconds or DD_TRACE_AGENT_TIMEOUT_SECONDS.
+      class AgentlessSettingsResolver < AgentSettingsResolver
+        # To avoid coupling this class to telemetry, the URL override is
+        # taken here as a parameter instead of being read out of
+        # c.telemetry.agentless_url_override. For the same reason, the
+        # +url_override_source+ parameter should be set to the string
+        # "c.telemetry.agentless_url_override".
+        def self.call(settings, host_prefix:, url_override: nil, url_override_source: nil, logger: Datadog.logger)
+          new(
+            settings,
+            host_prefix: host_prefix,
+            url_override: url_override,
+            url_override_source: url_override_source,
+            logger: logger
+          ).send(:call)
+        end
+
+        private
+
+        attr_reader \
+          :host_prefix,
+          :url_override,
+          :url_override_source
+
+        def initialize(settings, host_prefix:, url_override: nil, url_override_source: nil, logger: Datadog.logger)
+          if url_override && url_override_source.nil?
+            raise ArgumentError, 'url_override_source must be provided when url_override is provided'
+          end
+
+          super(settings, logger: logger)
+
+          @host_prefix = host_prefix
+          @url_override = url_override
+          @url_override_source = url_override_source
+        end
+
+        def hostname
+          if should_use_uds?
+            nil
+          else
+            configured_hostname || "#{host_prefix}.#{settings.site}"
+          end
+        end
+
+        def configured_hostname
+          return @configured_hostname if defined?(@configured_hostname)
+
+          if should_use_uds?
+            nil
+          else
+            @configured_hostname = (parsed_url.hostname if parsed_url)
+          end
+        end
+
+        def configured_port
+          return @configured_port if defined?(@configured_port)
+
+          @configured_port = (parsed_url.port if parsed_url)
+        end
+
+        # Note that this method should always return true or false
+        def ssl?
+          if configured_hostname
+            configured_ssl || false
+          else
+            if should_use_uds?
+              false
+            else
+              # If no hostname is specified, we are communicating with the
+              # default Datadog intake, which uses TLS.
+              true
+            end
+          end
+        end
+
+        # Note that this method can return nil
+        def configured_ssl
+          return @configured_ssl if defined?(@configured_ssl)
+
+          @configured_ssl = (parsed_url_ssl? if parsed_url)
+        end
+
+        def port
+          if configured_port
+            configured_port
+          else
+            if should_use_uds?
+              nil
+            else
+              # If no hostname is specified, we are communicating with the
+              # default Datadog intake, which exists on port 443.
+              443
+            end
+          end
+        end
+
+        def mixed_http_and_uds
+          false
+        end
+
+        def configured_uds_path
+          return @configured_uds_path if defined?(@configured_uds_path)
+
+          parsed_url_uds_path
+        end
+
+        def can_use_uds?
+          # While in theory agentless transport could communicate via UDS,
+          # in practice "agentless" means we are communicating with Datadog
+          # infrastructure which is always remote.
+          # Permit UDS for proxy usage?
+          !configured_uds_path.nil?
+        end
+
+        def parsed_url
+          return @parsed_url if defined?(@parsed_url)
+
+          @parsed_url =
+            if @url_override
+              parsed = URI.parse(@url_override)
+
+              # Agentless URL should never refer to a UDS?
+              if http_scheme?(parsed) || unix_scheme?(parsed)
+                parsed
+              else
+                log_warning(
+                  "Invalid URI scheme '#{parsed.scheme}' for #{url_override_source}. " \
+                  "Ignoring the contents of #{url_override_source}."
+                )
+                nil
+              end
+            end
+        end
+      end
+    end
+  end
+end
+
+# rubocop:enable Style/*

data/lib/datadog/core/configuration/components.rb

@@ -14,9 +14,11 @@ require_relative '../../tracing/component'
 require_relative '../../profiling/component'
 require_relative '../../appsec/component'
 require_relative '../../di/component'
+require_relative '../../error_tracking/component'
 require_relative '../crashtracking/component'
 
 require_relative '../environment/agent_info'
+require_relative '../process_discovery'
 
 module Datadog
   module Core
@@ -26,12 +28,12 @@ module Datadog
         class << self
           include Datadog::Tracing::Component
 
-          def build_health_metrics(settings, logger)
+          def build_health_metrics(settings, logger, telemetry)
             settings = settings.health_metrics
             options = { enabled: settings.enabled }
             options[:statsd] = settings.statsd unless settings.statsd.nil?
 
-            Core::Diagnostics::Health::Metrics.new(logger: logger, **options)
+            Core::Diagnostics::Health::Metrics.new(telemetry: telemetry, logger: logger, **options)
           end
 
           def build_logger(settings)
@@ -41,24 +43,24 @@ module Datadog
             logger
           end
 
-          def build_runtime_metrics(settings, logger)
+          def build_runtime_metrics(settings, logger, telemetry)
             options = { enabled: settings.runtime_metrics.enabled }
             options[:statsd] = settings.runtime_metrics.statsd unless settings.runtime_metrics.statsd.nil?
             options[:services] = [settings.service] unless settings.service.nil?
             options[:experimental_runtime_id_enabled] = settings.runtime_metrics.experimental_runtime_id_enabled
 
-            Core::Runtime::Metrics.new(logger: logger, **options)
+            Core::Runtime::Metrics.new(logger: logger, telemetry: telemetry, **options)
           end
 
-          def build_runtime_metrics_worker(settings, logger)
+          def build_runtime_metrics_worker(settings, logger, telemetry)
             # NOTE: Should we just ignore building the worker if its not enabled?
             options = settings.runtime_metrics.opts.merge(
               enabled: settings.runtime_metrics.enabled,
-              metrics: build_runtime_metrics(settings, logger),
+              metrics: build_runtime_metrics(settings, logger, telemetry),
               logger: logger,
             )
 
-            Core::Workers::RuntimeMetrics.new(options)
+            Core::Workers::RuntimeMetrics.new(telemetry: telemetry, **options)
           end
 
           def build_telemetry(settings, agent_settings, logger)
@@ -68,7 +70,7 @@ module Datadog
           def build_crashtracker(settings, agent_settings, logger:)
             return unless settings.crashtracking.enabled
 
-            if (libdatadog_api_failure = Datadog::Core::Crashtracking::Component::LIBDATADOG_API_FAILURE)
+            if (libdatadog_api_failure = Datadog::Core::LIBDATADOG_API_FAILURE)
               logger.debug("Cannot enable crashtracking: #{libdatadog_api_failure}")
               return
             end
@@ -88,6 +90,7 @@ module Datadog
           :telemetry,
           :tracer,
           :crashtracker,
+          :error_tracking,
           :dynamic_instrumentation,
           :appsec,
           :agent_info
@@ -118,11 +121,14 @@ module Datadog
           )
           @environment_logger_extra.merge!(profiler_logger_extra) if profiler_logger_extra
 
-          @runtime_metrics = self.class.build_runtime_metrics_worker(settings, @logger)
-          @health_metrics = self.class.build_health_metrics(settings, @logger)
+          @runtime_metrics = self.class.build_runtime_metrics_worker(settings, @logger, telemetry)
+          @health_metrics = self.class.build_health_metrics(settings, @logger, telemetry)
           @appsec = Datadog::AppSec::Component.build_appsec_component(settings, telemetry: telemetry)
           @dynamic_instrumentation = Datadog::DI::Component.build(settings, agent_settings, @logger, telemetry: telemetry)
+          @error_tracking = Datadog::ErrorTracking::Component.build(settings, @tracer, @logger)
           @environment_logger_extra[:dynamic_instrumentation_enabled] = !!@dynamic_instrumentation
+          # TODO: Re-enable this once we have updated libdatadog to 17.1
+          # @process_discovery_fd = Core::ProcessDiscovery.get_and_store_metadata(settings, @logger)
 
           self.class.configure_tracing(settings)
         end
@@ -203,6 +209,9 @@ module Datadog
           # enqueue closing event before stopping telemetry so it will be send out on shutdown
           telemetry.emit_closing! unless replacement
           telemetry.stop!
+
+          # TODO: Re-enable this once we have updated libdatadog to 17.1
+          # Core::ProcessDiscovery._native_close_tracer_memfd(@process_discovery_fd, @logger) if @process_discovery_fd
         end
       end
     end