datadog 2.4.0 → 2.44.0

data/lib/datadog/core/semaphore.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    # Semaphore pattern implementation, as described in documentation for
+    # ConditionVariable.
+    #
+    # @api private
+    class Semaphore
+      def initialize
+        @wake_lock = Mutex.new
+        @wake = ConditionVariable.new
+      end
+
+      def signal
+        wake_lock.synchronize do
+          wake.signal
+        end
+      end
+
+      def wait(timeout = nil)
+        wake_lock.synchronize do
+          wake.wait(wake_lock, timeout)
+        end
+      end
+
+      private
+
+      attr_reader :wake_lock, :wake
+    end
+  end
+end

data/lib/datadog/core/tag_builder.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require_relative "environment/socket"
+require_relative "environment/identity"
+require_relative "environment/git"
+
+module Datadog
+  module Core
+    # This module builds a hash of tags.
+    #
+    # @api private
+    module TagBuilder
+      def self.fixed_environment_tags
+        @fixed_environment_tags ||= {
+          "language" => Environment::Identity.lang,
+          "runtime" => Environment::Identity.lang, # Known to be duplicated from language above
+          "runtime_engine" => Environment::Identity.lang_engine,
+          "runtime_platform" => Environment::Identity.lang_platform,
+          "runtime_version" => Environment::Identity.lang_version,
+          "library_version" => Environment::Identity.gem_datadog_version,
+          "git.repository_url" => Environment::Git.git_repository_url,
+          "git.commit.sha" => Environment::Git.git_commit_sha,
+        }.compact.freeze
+      end
+
+      def self.tags(settings)
+        # Note that user tags get overwritten by our tags, and also
+        # that user tags do not get compacted (nil values are sent as
+        # empty strings).
+        settings.tags.merge(fixed_environment_tags).merge({
+          # Hostname can possibly change during application runtime.
+          "host" => Environment::Socket.hostname,
+          # Runtime ID changes upon a fork.
+          "runtime-id" => Environment::Identity.id,
+          # Process ID changes upon a fork.
+          "process_id" => Process.pid.to_s,
+          # Unified service tagging.
+          "env" => settings.env,
+          "service" => settings.service,
+          "version" => settings.version,
+        }.compact)
+      end
+
+      def self.serialize_tags(tags)
+        # DEV: Should there be some sort of escaping done here?
+        tags.map do |key, value|
+          "#{key}:#{value}"
+        end.join(",")
+      end
+    end
+  end
+end

data/lib/datadog/core/tag_normalizer.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require_relative "utils"
+
+module Datadog
+  module Core
+    # @api private
+    module TagNormalizer
+      # Normalization logic used for tag keys and values that the Trace Agent has for traces
+      # Useful for ensuring that tag keys and values are normalized consistently
+      # An use case for now is Process Tags which need to be sent across various intakes (profiling, tracing, etc.) consistently
+
+      module_function
+
+      INVALID_TAG_CHARACTERS = %r{[^\p{L}0-9_\-:./]}
+      LEADING_INVALID_CHARS_NO_DIGITS = %r{\A[^\p{L}:]++}
+      LEADING_INVALID_CHARS_WITH_DIGITS = %r{\A[^\p{L}0-9:./]++}
+      MAX_BYTE_SIZE = 200 # Represents the general max tag length
+      MAX_PROCESS_VALUE_BYTE_SIZE = 100 # Represents the max tag length for process tags
+      VALID_ASCII_TAG = %r{\A[a-z:][a-z0-9:./-]*\z}
+
+      # Based on https://github.com/DataDog/datadog-agent/blob/45799c842bbd216bcda208737f9f11cade6fdd95/pkg/trace/traceutil/normalize.go#L131
+      # Specifically for general normalization:
+      # - Must be valid UTF-8
+      # - Invalid characters are replaced with an underscore
+      # - Leading non-letter characters are removed but colons are kept
+      # - Trailing non-letter characters are removed
+      # - Trailing underscores are removed
+      # - Consecutive underscores are merged into a single underscore
+      # - Maximum length is 200 characters
+      # If it's a tag value, allow it to start with a digit
+      # @param original_value [String] The original string
+      # @param remove_digit_start_char [Boolean] - whether to remove the leading digit (currently only used for tag values)
+      # @return [String] The normalized string
+      def self.normalize(original_value, remove_digit_start_char: false)
+        # DEV-3.0: Ideally this encode call should be replaced with Datadog::Core::Utils.utf8_encode once it
+        #          is safe to modify the default behavior.
+        value = original_value.to_s.encode("UTF-8", invalid: :replace, undef: :replace)
+        value.strip!
+        return "" if value.empty?
+
+        return value if value.bytesize <= MAX_BYTE_SIZE &&
+          value.match?(VALID_ASCII_TAG)
+
+        if value.bytesize > MAX_BYTE_SIZE
+          value = value.byteslice(0, MAX_BYTE_SIZE)
+          value.scrub!("")
+        end
+
+        value.downcase!
+        value.gsub!(INVALID_TAG_CHARACTERS, "_")
+
+        # The Trace Agent allows tag values to start with a number so this logic is here too
+        leading_invalid_regex = remove_digit_start_char ? LEADING_INVALID_CHARS_NO_DIGITS : LEADING_INVALID_CHARS_WITH_DIGITS
+        value.sub!(leading_invalid_regex, "")
+
+        value.squeeze!("_") if value.include?("__")
+        value.delete_suffix!("_")
+
+        value
+      end
+
+      # Process tags values follow an additional piece of normalization:
+      # - must not be more than 100 bytes
+      # - and must not contain colons
+      # @param value [String] The original string
+      # @return [String] The normalized string
+      def self.normalize_process_value(value)
+        value = normalize(value)
+        return value if value.empty?
+
+        value.tr!(":", "_")
+        value.squeeze!("_") if value.include?("__")
+
+        if value.bytesize > MAX_PROCESS_VALUE_BYTE_SIZE
+          value = value.byteslice(0, MAX_PROCESS_VALUE_BYTE_SIZE) || value
+          value.scrub!("")
+        end
+
+        value
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/component.rb

@@ -1,21 +1,37 @@
 # frozen_string_literal: true
 
-require_relative 'emitter'
-require_relative 'event'
-require_relative 'http/transport'
-require_relative 'metrics_manager'
-require_relative 'worker'
-require_relative 'logging'
+require_relative "emitter"
+require_relative "event"
+require_relative "metrics_manager"
+require_relative "worker"
+require_relative "logging"
+require_relative "transport/http"
 
-require_relative '../configuration/ext'
-require_relative '../utils/forking'
+require_relative "../configuration/ext"
+require_relative "../configuration/agentless_settings_resolver"
+require_relative "../utils/forking"
 
 module Datadog
   module Core
     module Telemetry
-      # Telemetry entrypoint, coordinates sending telemetry events at various points in app lifecycle.
+      # Telemetry entry point, coordinates sending telemetry events at
+      # various points in application lifecycle.
+      #
+      # @api private
       class Component
+        ENDPOINT_COLLECTION_MESSAGE_LIMIT = 300
+
         attr_reader :enabled
+        attr_reader :logger
+        attr_reader :transport
+        attr_reader :worker
+        attr_reader :settings
+        attr_reader :agent_settings
+        attr_reader :metrics_manager
+
+        # Alias for consistency with other components.
+        # TODO Remove +enabled+ method
+        alias_method :enabled?, :enabled
 
         include Core::Utils::Forking
         include Telemetry::Logging
@@ -24,106 +40,157 @@ module Datadog
           enabled = settings.telemetry.enabled
           agentless_enabled = settings.telemetry.agentless_enabled
 
-          if !agentless_enabled && agent_settings.adapter != Datadog::Core::Configuration::Ext::Agent::HTTP::ADAPTER
-            enabled = false
-            logger.debug { "Telemetry disabled. Agent network adapter not supported: #{agent_settings.adapter}" }
-          end
-
           if agentless_enabled && settings.api_key.nil?
             enabled = false
-            logger.debug { 'Telemetry disabled. Agentless telemetry requires an DD_API_KEY variable to be set.' }
+            logger.debug { "Telemetry disabled. Agentless telemetry requires a DD_API_KEY variable to be set." }
           end
 
-          transport = if agentless_enabled
-                        Datadog::Core::Telemetry::Http::Transport.build_agentless_transport(
-                          api_key: settings.api_key,
-                          dd_site: settings.site,
-                          url_override: settings.telemetry.agentless_url_override
-                        )
-                      else
-                        Datadog::Core::Telemetry::Http::Transport.build_agent_transport(agent_settings)
-                      end
-
           Telemetry::Component.new(
-            http_transport: transport,
+            settings: settings,
+            agent_settings: agent_settings,
             enabled: enabled,
-            metrics_enabled: enabled && settings.telemetry.metrics_enabled,
-            heartbeat_interval_seconds: settings.telemetry.heartbeat_interval_seconds,
-            metrics_aggregation_interval_seconds: settings.telemetry.metrics_aggregation_interval_seconds,
-            dependency_collection: settings.telemetry.dependency_collection,
-            shutdown_timeout_seconds: settings.telemetry.shutdown_timeout_seconds,
+            logger: logger,
           )
         end
 
         # @param enabled [Boolean] Determines whether telemetry events should be sent to the API
-        # @param metrics_enabled [Boolean] Determines whether telemetry metrics should be sent to the API
-        # @param heartbeat_interval_seconds [Float] How frequently heartbeats will be reported, in seconds.
-        # @param metrics_aggregation_interval_seconds [Float] How frequently metrics will be aggregated, in seconds.
-        # @param [Boolean] dependency_collection Whether to send the `app-dependencies-loaded` event
-        def initialize(
-          heartbeat_interval_seconds:,
-          metrics_aggregation_interval_seconds:,
-          dependency_collection:,
-          http_transport:,
-          shutdown_timeout_seconds:,
-          enabled: true,
-          metrics_enabled: true
+        def initialize( # standard:disable Metrics/MethodLength
+          settings:,
+          agent_settings:,
+          logger:,
+          enabled:
         )
           @enabled = enabled
-          @stopped = false
+          @log_collection_enabled = settings.telemetry.log_collection_enabled
+          @logger = logger
 
           @metrics_manager = MetricsManager.new(
-            enabled: enabled && metrics_enabled,
-            aggregation_interval: metrics_aggregation_interval_seconds
+            enabled: @enabled && settings.telemetry.metrics_enabled,
+            aggregation_interval: settings.telemetry.metrics_aggregation_interval_seconds,
           )
 
+          @stopped = false
+
+          return unless @enabled
+
+          @transport = if settings.telemetry.agentless_enabled
+            # We don't touch the `agent_settings` since we still want the telemetry payloads to refer to the original
+            # settings, even though the telemetry itself may be using a different path.
+            telemetry_specific_agent_settings = Core::Configuration::AgentlessSettingsResolver.call(
+              settings,
+              host_prefix: "instrumentation-telemetry-intake",
+              url_override: settings.telemetry.agentless_url_override,
+              url_override_source: "c.telemetry.agentless_url_override",
+              logger: logger,
+            )
+            Telemetry::Transport::HTTP.agentless_telemetry(
+              agent_settings: telemetry_specific_agent_settings,
+              logger: logger,
+              # api_key should have already validated to be
+              # not nil by +build+ method above.
+              api_key: settings.api_key,
+            )
+          else
+            Telemetry::Transport::HTTP.agent_telemetry(
+              agent_settings: agent_settings, logger: logger,
+            )
+          end
+
           @worker = Telemetry::Worker.new(
             enabled: @enabled,
-            heartbeat_interval_seconds: heartbeat_interval_seconds,
-            metrics_aggregation_interval_seconds: metrics_aggregation_interval_seconds,
-            emitter: Emitter.new(http_transport: http_transport),
+            heartbeat_interval_seconds: settings.telemetry.heartbeat_interval_seconds,
+            extended_heartbeat_interval_seconds: settings.telemetry.extended_heartbeat_interval_seconds,
+            metrics_aggregation_interval_seconds: settings.telemetry.metrics_aggregation_interval_seconds,
+            emitter: Emitter.new(
+              @transport,
+              logger: @logger,
+              debug: settings.telemetry.debug,
+            ),
             metrics_manager: @metrics_manager,
-            dependency_collection: dependency_collection,
-            shutdown_timeout: shutdown_timeout_seconds
+            dependency_collection: settings.telemetry.dependency_collection,
+            logger: logger,
+            settings: settings,
+            agent_settings: agent_settings,
+            shutdown_timeout: settings.telemetry.shutdown_timeout_seconds,
           )
-          @worker.start
+
+          @agent_settings = agent_settings
         end
 
         def disable!
           @enabled = false
-          @worker.enabled = false
+          @worker&.enabled = false
+        end
+
+        def start(initial_event_is_change = false, components:)
+          return unless enabled?
+
+          initial_event = if initial_event_is_change
+            Event::SynthAppClientConfigurationChange.new(
+              components: components,
+            )
+          else
+            Event::AppStarted.new(
+              components: components,
+            )
+          end
+
+          @worker.start(initial_event)
         end
 
-        def stop!
+        def shutdown!
           return if @stopped
 
-          @worker.stop(true)
+          if defined?(@worker)
+            @worker&.stop(true)
+          end
+
           @stopped = true
         end
 
         def emit_closing!
-          return if !@enabled || forked?
+          return unless enabled?
 
           @worker.enqueue(Event::AppClosing.new)
         end
 
         def integrations_change!
-          return if !@enabled || forked?
+          return unless enabled?
 
           @worker.enqueue(Event::AppIntegrationsChange.new)
         end
 
         def log!(event)
-          return unless @enabled || forked?
+          return unless enabled? && @log_collection_enabled
 
           @worker.enqueue(event)
         end
 
+        # Wait for the worker to send out all events that have already
+        # been queued, up to 15 seconds. Returns whether all events have
+        # been flushed, or nil if telemetry is disabled.
+        #
+        # @api private
+        def flush(timeout: nil)
+          return unless enabled?
+
+          @worker.flush(timeout: timeout)
+        end
+
         # Report configuration changes caused by Remote Configuration.
         def client_configuration_change!(changes)
-          return if !@enabled || forked?
+          return unless enabled?
+
+          @worker.enqueue(Event::AppClientConfigurationChange.new(changes, "remote_config"))
+        end
 
-          @worker.enqueue(Event::AppClientConfigurationChange.new(changes, 'remote_config'))
+        # Report application endpoints
+        def app_endpoints_loaded(endpoints, page_size: ENDPOINT_COLLECTION_MESSAGE_LIMIT)
+          return unless enabled?
+
+          endpoints.each_slice(page_size).with_index do |endpoints_slice, i|
+            @worker.enqueue(Event::AppEndpointsLoaded.new(endpoints_slice, is_first: i.zero?))
+          end
         end
 
         # Increments a count metric.
@@ -150,6 +217,22 @@ module Datadog
         def distribution(namespace, metric_name, value, tags: {}, common: true)
           @metrics_manager.distribution(namespace, metric_name, value, tags: tags, common: common)
         end
+
+        # When a fork happens, we generally need to do two things inside the
+        # child proess:
+        # 1. Restart the worker.
+        # 2. Discard any events and metrics that were submitted in the
+        #    parent process (because they will be sent out in the parent
+        #    process, sending them in the child would cause duplicate
+        #    submission).
+        def after_fork
+          # We cannot simply create a new instance of metrics manager because
+          # it is referenced from other objects (e.g. the worker).
+          # We must reset the existing instance.
+          @metrics_manager.clear
+
+          worker&.send(:after_fork_monkey_patched)
+        end
       end
     end
   end

data/lib/datadog/core/telemetry/configuration_value.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Telemetry
+      module ConfigurationValue
+        module_function
+
+        def convert(value)
+          case value
+          when Integer, String, true, false, nil
+            value
+          when Float
+            value.finite? ? value : value.to_s
+          when Hash
+            value.map { |key, entry_value| "#{key}:#{entry_value}" }.join(",")
+          when Array
+            value.join(",")
+          when Module
+            value.name.to_s
+          else
+            implements_to_s = begin
+              value.method(:to_s).owner != Kernel
+            rescue NameError
+              false
+            end
+
+            if implements_to_s
+              value.to_s
+            else
+              value.class.to_s
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/emitter.rb

@@ -1,35 +1,47 @@
 # frozen_string_literal: true
 
-require_relative 'request'
-require_relative 'http/transport'
-require_relative '../utils/sequence'
-require_relative '../utils/forking'
+require_relative "request"
+require_relative "../transport/response"
+require_relative "../utils/sequence"
+require_relative "../utils/forking"
 
 module Datadog
   module Core
     module Telemetry
       # Class that emits telemetry events
       class Emitter
-        attr_reader :http_transport
+        attr_reader :transport, :logger
 
         extend Core::Utils::Forking
 
-        # @param http_transport [Datadog::Core::Telemetry::Http::Transport] Transport object that can be used to send
-        #   telemetry requests via the agent
-        def initialize(http_transport:)
-          @http_transport = http_transport
+        # @param transport [Datadog::Core::Telemetry::Transport::Telemetry::Transport]
+        #   Transport object that can be used to send telemetry requests
+        def initialize(transport, logger: Datadog.logger, debug: false)
+          @transport = transport
+          @logger = logger
+          @debug = !!debug
+        end
+
+        def debug?
+          @debug
         end
 
         # Retrieves and emits a TelemetryRequest object based on the request type specified
         def request(event)
           seq_id = self.class.sequence.next
-          payload = Request.build_payload(event, seq_id)
-          res = @http_transport.request(request_type: event.type, payload: payload.to_json)
-          Datadog.logger.debug { "Telemetry sent for event `#{event.type}` (code: #{res.code.inspect})" }
+          payload = Request.build_payload(event, seq_id, debug: debug?)
+          res = @transport.send_telemetry(request_type: event.type, payload: payload)
+          if res.ok?
+            logger.debug { "Telemetry sent for event `#{event.type}`" }
+          else
+            logger.debug { "Failed to send telemetry for event `#{event.type}`: #{res.inspect}" }
+          end
           res
         rescue => e
-          Datadog.logger.debug("Unable to send telemetry request for event `#{event.type rescue 'unknown'}`: #{e}")
-          Telemetry::Http::InternalErrorResponse.new(e)
+          logger.debug {
+            "Unable to send telemetry request for event `#{event.respond_to?(:type) ? event.type : event.to_s}`: #{e.class}: #{e.message}"
+          }
+          Core::Transport::InternalErrorResponse.new(e)
         end
 
         # Initializes a Sequence object to track seq_id if not already initialized; else returns stored

data/lib/datadog/core/telemetry/event/app_client_configuration_change.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require_relative "base"
+require_relative "../configuration_value"
+
+module Datadog
+  module Core
+    module Telemetry
+      module Event
+        # Telemetry class for the 'app-client-configuration-change' event
+        class AppClientConfigurationChange < Base
+          attr_reader :changes, :origin
+
+          def type
+            "app-client-configuration-change"
+          end
+
+          def initialize(changes, origin)
+            super()
+            @changes = changes
+            @origin = origin
+          end
+
+          def payload
+            {configuration: configuration}
+          end
+
+          def configuration
+            config = Datadog.configuration
+
+            # @type var res: Array[telemetry_configuration]
+            res = @changes.map do |name, value|
+              {
+                name: name,
+                value: Telemetry::ConfigurationValue.convert(value),
+                origin: @origin,
+                seq_id: Configuration::Option::Precedence::REMOTE_CONFIGURATION.numeric.next,
+              }
+            end
+
+            # DEV: This seems unnecessary (we send the state of sca_enabled for each remote config change)
+            unless config.dig("appsec", "sca_enabled").nil?
+              res << {
+                name: "appsec.sca_enabled",
+                value: config.appsec.sca_enabled,
+                origin: "code",
+                seq_id: Configuration::Option::Precedence::PROGRAMMATIC.numeric.next,
+              }
+            end
+
+            res
+          end
+
+          def ==(other)
+            other.is_a?(AppClientConfigurationChange) && other.changes == @changes && other.origin == @origin
+          end
+
+          alias_method :eql?, :==
+
+          def hash
+            [self.class, @changes, @origin].hash
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/event/app_closing.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module Datadog
+  module Core
+    module Telemetry
+      module Event
+        # Telemetry class for the 'app-closing' event
+        class AppClosing < Base
+          def type
+            "app-closing"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/event/app_dependencies_loaded.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module Datadog
+  module Core
+    module Telemetry
+      module Event
+        # Telemetry class for the 'app-dependencies-loaded' event
+        class AppDependenciesLoaded < Base
+          def type
+            "app-dependencies-loaded"
+          end
+
+          def payload
+            {dependencies: dependencies}
+          end
+
+          private
+
+          def dependencies
+            Gem.loaded_specs.collect do |name, gem|
+              {
+                name: name,
+                version: gem.version.to_s,
+              }
+            end
+          end
+        end
+      end
+    end
+  end
+end