datadog 2.0.0.beta1 → 2.0.0.beta2

data/lib/datadog/appsec/contrib/rack/request_middleware.rb

@@ -15,6 +15,18 @@ module Datadog
   module AppSec
     module Contrib
       module Rack
+        # Create an array of lowercased headers
+        WAF_VENDOR_HEADERS_TAGS = %w[
+          X-Amzn-Trace-Id
+          Cloudfront-Viewer-Ja3-Fingerprint
+          Cf-Ray
+          X-Cloud-Trace-Context
+          X-Appgw-Trace-id
+          X-SigSci-RequestID
+          X-SigSci-Tags
+          Akamai-User-Risk
+        ].map(&:downcase).freeze
+
         # Topmost Rack middleware for AppSec
         # This should be inserted just below Datadog::Tracing::Contrib::Rack::TraceMiddleware
         class RequestMiddleware
@@ -22,6 +34,7 @@ module Datadog
             @app = app
 
             @oneshot_tags_sent = false
+            @rack_headers = {}
           end
 
           # rubocop:disable Metrics/AbcSize,Metrics/PerceivedComplexity,Metrics/CyclomaticComplexity,Metrics/MethodLength
@@ -56,7 +69,8 @@ module Datadog
 
             gateway_request = Gateway::Request.new(env)
 
-            add_appsec_tags(processor, scope, env)
+            add_appsec_tags(processor, scope)
+            add_request_tags(scope, env)
 
             request_return, request_response = catch(::Datadog::AppSec::Ext::INTERRUPT) do
               Instrumentation.gateway.push('rack.request', gateway_request) do
@@ -131,7 +145,7 @@ module Datadog
             Datadog::Tracing.active_span
           end
 
-          def add_appsec_tags(processor, scope, env)
+          def add_appsec_tags(processor, scope)
             span = scope.service_entry_span
             trace = scope.trace
 
@@ -141,17 +155,6 @@ module Datadog
             span.set_tag('_dd.runtime_family', 'ruby')
             span.set_tag('_dd.appsec.waf.version', Datadog::AppSec::WAF::VERSION::BASE_STRING)
 
-            if span && span.get_tag(Tracing::Metadata::Ext::HTTP::TAG_CLIENT_IP).nil?
-              request_header_collection = Datadog::Tracing::Contrib::Rack::Header::RequestHeaderCollection.new(env)
-
-              # always collect client ip, as this is part of AppSec provided functionality
-              Datadog::Tracing::ClientIp.set_client_ip_tag!(
-                span,
-                headers: request_header_collection,
-                remote_ip: env['REMOTE_ADDR']
-              )
-            end
-
             if processor.diagnostics
               diagnostics = processor.diagnostics
 
@@ -177,6 +180,29 @@ module Datadog
             end
           end
 
+          def add_request_tags(scope, env)
+            span = scope.service_entry_span
+
+            return unless span
+
+            # Always add WAF vendors headers
+            WAF_VENDOR_HEADERS_TAGS.each do |lowercase_header|
+              rack_header = to_rack_header(lowercase_header)
+              span.set_tag("http.request.headers.#{lowercase_header}", env[rack_header]) if env[rack_header]
+            end
+
+            if span && span.get_tag(Tracing::Metadata::Ext::HTTP::TAG_CLIENT_IP).nil?
+              request_header_collection = Datadog::Tracing::Contrib::Rack::Header::RequestHeaderCollection.new(env)
+
+              # always collect client ip, as this is part of AppSec provided functionality
+              Datadog::Tracing::ClientIp.set_client_ip_tag!(
+                span,
+                headers: request_header_collection,
+                remote_ip: env['REMOTE_ADDR']
+              )
+            end
+          end
+
           def add_waf_runtime_tags(scope)
             span = scope.service_entry_span
             context = scope.processor_context
@@ -189,6 +215,10 @@ module Datadog
             span.set_tag('_dd.appsec.waf.duration', context.time_ns / 1000.0)
             span.set_tag('_dd.appsec.waf.duration_ext', context.time_ext_ns / 1000.0)
           end
+
+          def to_rack_header(header)
+            @rack_headers[header] ||= Datadog::Tracing::Contrib::Rack::Header.to_rack_header(header)
+          end
         end
       end
     end

data/lib/datadog/appsec/event.rb

@@ -2,9 +2,9 @@
 
 require 'json'
 require 'zlib'
-require 'base64'
 
 require_relative 'rate_limiter'
+require_relative '../core/utils/base64'
 
 module Datadog
   module AppSec
@@ -140,7 +140,7 @@ module Datadog
         private
 
         def compressed_and_base64_encoded(value)
-          Base64.encode64(gzip(value))
+          Datadog::Core::Utils::Base64.strict_encode64(gzip(value))
         rescue TypeError => e
           Datadog.logger.debug do
             "Failed to compress and encode value when populating AppSec::Event. Error: #{e.message}"

data/lib/datadog/core/configuration/components.rb

@@ -64,7 +64,8 @@ module Datadog
 
             Telemetry::Client.new(
               enabled: enabled,
-              heartbeat_interval_seconds: settings.telemetry.heartbeat_interval_seconds
+              heartbeat_interval_seconds: settings.telemetry.heartbeat_interval_seconds,
+              dependency_collection: settings.telemetry.dependency_collection
             )
           end
         end

data/lib/datadog/core/configuration/option.rb

@@ -8,7 +8,13 @@ module Datadog
       # Represents an instance of an integration configuration option
       # @public_api
       class Option
-        attr_reader :definition
+        # @!attribute [r] definition
+        #   The definition object that matches this option.
+        #   @return [Configuration::OptionDefinition]
+        # @!attribute [r] precedence_set
+        #   When this option was last set, what was the value precedence used?
+        #   @return [Precedence::Value]
+        attr_reader :definition, :precedence_set
 
         # Option setting precedence.
         module Precedence
@@ -305,10 +311,6 @@ module Datadog
                               "but '#{ENV[effective_env]}' was provided"
         end
 
-        # Used for testing
-        attr_reader :precedence_set
-        private :precedence_set
-
         # Anchor object that represents a value that is not set.
         # This is necessary because `nil` is a valid value to be set.
         UNSET = Object.new

data/lib/datadog/core/configuration/settings.rb

@@ -271,29 +271,43 @@ module Datadog
 
             # Can be used to disable the gathering of names and versions of gems in use by the service, used to power
             # grouping and categorization of stack traces.
-            option :code_provenance_enabled, default: true
+            option :code_provenance_enabled do |o|
+              o.type :bool
+              o.default true
+            end
 
             # Forces enabling of profiling of time/resources spent in Garbage Collection.
             #
             # Note that setting this to "false" (or not setting it) will not prevent the feature from being
             # being automatically enabled in the future.
             #
-            # This feature defaults to off for two reasons:
-            # 1. Currently this feature can add a lot of overhead for GC-heavy workloads.
-            # 2. Although this feature is safe on Ruby 2.x, on Ruby 3.x it can break in applications that make use of
-            #    Ractors due to two Ruby VM bugs:
-            #    https://bugs.ruby-lang.org/issues/19112 AND https://bugs.ruby-lang.org/issues/18464.
-            #    If you use Ruby 3.x and your application does not use Ractors (or if your Ruby has been patched), the
-            #    feature is fully safe to enable and this toggle can be used to do so.
+            # GC profiling is now on by default and controlled by {:gc_enabled}.
+            option :force_enable_gc_profiling do |o|
+              o.after_set do |_, _, precedence|
+                unless precedence == Datadog::Core::Configuration::Option::Precedence::DEFAULT
+                  Datadog.logger.warn(
+                    'The profiling.advanced.force_enable_gc_profiling setting has been deprecated for removal and no ' \
+                    'longer does anything (the feature is now on by default). ' \
+                    'Please remove this setting from your Datadog.configure block.'
+                  )
+                end
+              end
+            end
+
+            # Can be used to enable/disable garbage collection profiling.
             #
-            # We expect the once the above issues are overcome, we'll automatically enable the feature on fixed Ruby
-            # versions.
+            # @warn To avoid https://bugs.ruby-lang.org/issues/18464 even when enabled, GC profiling is only started
+            #       for Ruby versions 2.x, 3.1.4+, 3.2.3+ and 3.3.0+
+            #       (more details in {Datadog::Profiling::Component.enable_gc_profiling?})
             #
-            # @default `DD_PROFILING_FORCE_ENABLE_GC` environment variable, otherwise `false`
-            option :force_enable_gc_profiling do |o|
-              o.env 'DD_PROFILING_FORCE_ENABLE_GC'
+            # @warn Due to a VM bug in the Ractor implementation (https://bugs.ruby-lang.org/issues/19112) this feature
+            #       stops working when Ractors get garbage collected.
+            #
+            # @default `DD_PROFILING_GC_ENABLED` environment variable, otherwise `true`
+            option :gc_enabled do |o|
               o.type :bool
-              o.default false
+              o.env 'DD_PROFILING_GC_ENABLED'
+              o.default true
             end
 
             # Can be used to enable/disable the Datadog::Profiling.allocation_count feature.
@@ -646,6 +660,16 @@ module Datadog
         # Client-side telemetry configuration
         # @public_api
         settings :telemetry do
+          # Whether the bundled Ruby gems as reported through telemetry.
+          #
+          # @default `DD_TELEMETRY_DEPENDENCY_COLLECTION_ENABLED` environment variable, otherwise `true`.
+          # @return [Boolean]
+          option :dependency_collection do |o|
+            o.type :bool
+            o.env Core::Telemetry::Ext::ENV_DEPENDENCY_COLLECTION
+            o.default true
+          end
+
           # Enable telemetry collection. This allows telemetry events to be emitted to the telemetry API.
           #
           # @default `DD_INSTRUMENTATION_TELEMETRY_ENABLED` environment variable, otherwise `true`.

data/lib/datadog/core/configuration.rb

@@ -84,18 +84,23 @@ module Datadog
         configuration = self.configuration
         yield(configuration)
 
-        safely_synchronize do |write_components|
+        built_components = false
+
+        components = safely_synchronize do |write_components|
           write_components.call(
             if components?
               replace_components!(configuration, @components)
             else
               components = build_components(configuration)
-              components.telemetry.started!
+              built_components = true
               components
             end
           )
         end
 
+        # Should only be called the first time components are built
+        components.telemetry.started! if built_components
+
         configuration
       end
 
@@ -195,9 +200,20 @@ module Datadog
         current_components = COMPONENTS_READ_LOCK.synchronize { defined?(@components) && @components }
         return current_components if current_components || !allow_initialization
 
-        safely_synchronize do |write_components|
-          (defined?(@components) && @components) || write_components.call(build_components(configuration))
+        built_components = false
+
+        components = safely_synchronize do |write_components|
+          if defined?(@components) && @components
+            @components
+          else
+            built_components = true
+            write_components.call(build_components(configuration))
+          end
         end
+
+        # Should only be called the first time components are built
+        components&.telemetry&.started! if built_components
+        components
       end
 
       private

data/lib/datadog/core/environment/platform.rb

@@ -11,12 +11,18 @@ module Datadog
       module Platform
         module_function
 
+        # @return [String] ISA of host; `uname -m`
+        def architecture
+          Identity.lang_version >= '2.2' ? Etc.uname[:machine] : Gem::Platform.local.cpu
+        end
+
         # @return [String] name of host; `uname -n`
         def hostname
           Identity.lang_version >= '2.2' ? Etc.uname[:nodename] : nil
         end
 
-        # @return [String] name of kernel; `uname -s`
+        # System name, normally `Linux` or `Darwin` (but 'Mac OS X' on JRuby);
+        # @return [String] name of kernel; `uname -s`.
         def kernel_name
           Identity.lang_version >= '2.2' ? Etc.uname[:sysname] : Gem::Platform.local.os.capitalize
         end

data/lib/datadog/core/remote/client/capabilities.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative '../../utils/base64'
 require_relative '../../../appsec/remote'
 require_relative '../../../tracing/remote'
 
@@ -53,7 +54,7 @@ module Datadog
             cap_to_hexs = capabilities.reduce(:|).to_s(16).tap { |s| s.size.odd? && s.prepend('0') }.scan(/\h\h/)
             binary = cap_to_hexs.each_with_object([]) { |hex, acc| acc << hex }.map { |e| e.to_i(16) }.pack('C*')
 
-            Base64.encode64(binary).chomp
+            Datadog::Core::Utils::Base64.strict_encode64(binary)
           end
         end
       end

data/lib/datadog/core/remote/transport/http/config.rb

@@ -1,10 +1,10 @@
 # frozen_string_literal: true
 
 require 'json'
-require 'base64'
 
 require_relative '../config'
 require_relative 'client'
+require_relative '../../../utils/base64'
 require_relative '../../../transport/http/response'
 require_relative '../../../transport/http/api/endpoint'
 
@@ -51,7 +51,7 @@ module Datadog
 
                 # TODO: these fallbacks should be improved
                 roots = payload[:roots] || []
-                targets = payload[:targets] || Base64.encode64('{}').chomp
+                targets = payload[:targets] || Datadog::Core::Utils::Base64.strict_encode64('{}')
                 target_files = payload[:target_files] || []
                 client_configs = payload[:client_configs] || []
 
@@ -61,7 +61,7 @@ module Datadog
                   raise TypeError.new(String, root) unless root.is_a?(String)
 
                   decoded = begin
-                    Base64.strict_decode64(root) # TODO: unprocessed, don't symbolize_names
+                    Datadog::Core::Utils::Base64.strict_decode64(root) # TODO: unprocessed, don't symbolize_names
                   rescue ArgumentError
                     raise DecodeError.new(:roots, root)
                   end
@@ -81,7 +81,7 @@ module Datadog
 
                 @targets = begin
                   decoded = begin
-                    Base64.strict_decode64(targets)
+                    Datadog::Core::Utils::Base64.strict_decode64(targets)
                   rescue ArgumentError
                     raise DecodeError.new(:targets, targets)
                   end
@@ -109,7 +109,7 @@ module Datadog
                   raise TypeError.new(String, raw) unless raw.is_a?(String)
 
                   content = begin
-                    Base64.strict_decode64(raw)
+                    Datadog::Core::Utils::Base64.strict_decode64(raw)
                   rescue ArgumentError
                     raise DecodeError.new(:target_files, raw)
                   end

data/lib/datadog/core/telemetry/client.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative 'emitter'
+require_relative 'event'
 require_relative 'heartbeat'
 require_relative '../utils/forking'
 
@@ -10,21 +11,25 @@ module Datadog
       # Telemetry entrypoint, coordinates sending telemetry events at various points in app lifecycle.
       class Client
         attr_reader \
-          :emitter,
           :enabled,
-          :unsupported,
-          :worker
+          :unsupported
 
         include Core::Utils::Forking
 
         # @param enabled [Boolean] Determines whether telemetry events should be sent to the API
         # @param heartbeat_interval_seconds [Float] How frequently heartbeats will be reported, in seconds.
-        def initialize(heartbeat_interval_seconds:, enabled: true)
+        # @param [Boolean] dependency_collection Whether to send the `app-dependencies-loaded` event
+        def initialize(heartbeat_interval_seconds:, dependency_collection:, enabled: true)
           @enabled = enabled
           @emitter = Emitter.new
           @stopped = false
           @unsupported = false
+          @started = false
+          @dependency_collection = dependency_collection
+
           @worker = Telemetry::Heartbeat.new(enabled: @enabled, heartbeat_interval_seconds: heartbeat_interval_seconds) do
+            next unless @started # `started!` should be the first event, thus ensure that `heartbeat!` is not sent first.
+
             heartbeat!
           end
         end
@@ -37,21 +42,24 @@ module Datadog
         def started!
           return if !@enabled || forked?
 
-          res = @emitter.request(:'app-started')
+          res = @emitter.request(Event::AppStarted.new)
 
           if res.not_found? # Telemetry is only supported by agent versions 7.34 and up
             Datadog.logger.debug('Agent does not support telemetry; disabling future telemetry events.')
             disable!
             @unsupported = true # Prevent telemetry from getting re-enabled
+            return res
           end
 
-          res
+          @emitter.request(Event::AppDependenciesLoaded.new) if @dependency_collection
+
+          @started = true
         end
 
         def emit_closing!
           return if !@enabled || forked?
 
-          @emitter.request(:'app-closing')
+          @emitter.request(Event::AppClosing.new)
         end
 
         def stop!
@@ -64,14 +72,14 @@ module Datadog
         def integrations_change!
           return if !@enabled || forked?
 
-          @emitter.request(:'app-integrations-change')
+          @emitter.request(Event::AppIntegrationsChange.new)
         end
 
         # Report configuration changes caused by Remote Configuration.
         def client_configuration_change!(changes)
           return if !@enabled || forked?
 
-          @emitter.request('app-client-configuration-change', data: { changes: changes, origin: 'remote_config' })
+          @emitter.request(Event::AppClientConfigurationChange.new(changes, 'remote_config'))
         end
 
         private
@@ -79,7 +87,7 @@ module Datadog
         def heartbeat!
           return if !@enabled || forked?
 
-          @emitter.request(:'app-heartbeat')
+          @emitter.request(Event::AppHeartbeat.new)
         end
       end
     end

data/lib/datadog/core/telemetry/emitter.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative 'event'
+require_relative 'request'
 require_relative 'http/transport'
 require_relative '../utils/sequence'
 require_relative '../utils/forking'
@@ -14,7 +14,6 @@ module Datadog
 
         extend Core::Utils::Forking
 
-        # @param sequence [Datadog::Core::Utils::Sequence] Sequence object that stores and increments a counter
         # @param http_transport [Datadog::Core::Telemetry::Http::Transport] Transport object that can be used to send
         #   telemetry requests via the agent
         def initialize(http_transport: Datadog::Core::Telemetry::Http::Transport.new)
@@ -22,18 +21,15 @@ module Datadog
         end
 
         # Retrieves and emits a TelemetryRequest object based on the request type specified
-        # @param request_type [String] the type of telemetry request to collect data for
-        # @param data [Object] arbitrary object to be passed to the respective `request_type` handler
-        def request(request_type, data: nil)
+        def request(event)
           begin
-            request = Datadog::Core::Telemetry::Event.new.telemetry_request(
-              request_type: request_type,
-              seq_id: self.class.sequence.next,
-              data: data,
-            ).to_h
-            @http_transport.request(request_type: request_type.to_s, payload: request.to_json)
-          rescue StandardError => e
-            Datadog.logger.debug("Unable to send telemetry request for event `#{request_type}`: #{e}")
+            seq_id = self.class.sequence.next
+            payload = Request.build_payload(event, seq_id)
+            res = @http_transport.request(request_type: event.type, payload: payload.to_json)
+            Datadog.logger.debug { "Telemetry sent for event `#{event.type}` (status code: #{res.code})" }
+            res
+          rescue => e
+            Datadog.logger.debug("Unable to send telemetry request for event `#{event.type rescue 'unknown'}`: #{e}")
             Telemetry::Http::InternalErrorResponse.new(e)
           end
         end