ddtrace 1.12.1 → 1.13.0

data/lib/datadog/core/remote/component.rb

@@ -45,6 +45,11 @@ module Datadog
                 "remote worker client sync error: #{e.message} location: #{Array(e.backtrace).first}. skipping sync"
               end
             rescue StandardError => e
+              # In case of unexpected errors, reset the negotiation object
+              # given external conditions have changed and the negotiation
+              # negotiation object stores error logging state that should be reset.
+              negotiation = Negotiation.new(settings, agent_settings)
+
               Datadog.logger.error do
                 "remote worker error: #{e.class.name} #{e.message} location: #{Array(e.backtrace).first}. "\
                 'reseting client state'
@@ -62,13 +67,22 @@ module Datadog
           end
         end
 
-        def barrier(kind)
+        # Starts the Remote Configuration worker without waiting for first run
+        def start
           @worker.start
+        end
 
-          case kind
-          when :once
-            @barrier.wait_once
-          end
+        # Is the Remote Configuration worker running?
+        def started?
+          @worker.started?
+        end
+
+        # If the worker is not initialized, initialize it.
+        #
+        # Then, waits for one client sync to be executed if `kind` is `:once`.
+        def barrier(_kind)
+          start
+          @barrier.wait_once
         end
 
         def shutdown!
@@ -105,18 +119,6 @@ module Datadog
             end
           end
 
-          # Wait for next lift to happen
-          def wait_next(timeout = nil)
-            @mutex.lock
-
-            timeout ||= @timeout
-
-            # rbs/core has a bug, timeout type is incorrectly ?Integer
-            @condition.wait(@mutex, _ = timeout)
-          ensure
-            @mutex.unlock
-          end
-
           # Release all current waiters
           def lift
             @mutex.lock
@@ -130,26 +132,15 @@ module Datadog
         end
 
         class << self
+          # Because the agent might not be available yet, we can't perform agent-specific checks yet, as they
+          # would prevent remote configuration from ever running.
+          #
+          # Those checks are instead performed inside the worker loop.
+          # This allows users to upgrade their agent while keeping their application running.
           def build(settings, agent_settings)
             return unless settings.remote.enabled
 
-            capabilities = Client::Capabilities.new(settings)
-
-            return if capabilities.products.empty?
-
-            negotiation = Negotiation.new(settings, agent_settings)
-
-            unless negotiation.endpoint?('/v0.7/config')
-              Datadog.logger.error do
-                'endpoint unavailable: disabling remote configuration for this process.'
-              end
-
-              return
-            end
-
-            Datadog.logger.debug { 'agent reachable and reports remote configuration endpoint' }
-
-            new(settings, capabilities, agent_settings)
+            new(settings, Client::Capabilities.new(settings), agent_settings)
           end
         end
       end

data/lib/datadog/core/remote/configuration/content.rb

@@ -18,12 +18,14 @@ module Datadog
             end
           end
 
-          attr_reader :path, :data, :hashes
+          attr_reader :path, :data, :hashes, :apply_state, :apply_error
           attr_accessor :version
 
           def initialize(path:, data:)
             @path = path
             @data = data
+            @apply_state = ApplyState::UNACKNOWLEDGED
+            @apply_error = nil
             @hashes = {}
             @version = 0
           end
@@ -36,6 +38,31 @@ module Datadog
             @length ||= @data.size
           end
 
+          # Sets this configuration as successfully applied.
+          def applied
+            @apply_state = ApplyState::ACKNOWLEDGED
+            @apply_error = nil
+          end
+
+          # Sets this configuration as not successfully applied, with
+          # a message describing the error.
+          def errored(error_message)
+            @apply_state = ApplyState::ERROR
+            @apply_error = error_message
+          end
+
+          module ApplyState
+            # Default state of configurations.
+            # Set until the component consuming the configuration has acknowledged it was applied.
+            UNACKNOWLEDGED = 1
+
+            # Set when the configuration has been successfully applied.
+            ACKNOWLEDGED = 2
+
+            # Set when the configuration has been unsuccessfully applied.
+            ERROR = 3
+          end
+
           private
 
           def compute_and_store_hash(type)

data/lib/datadog/core/remote/configuration/repository.rb

@@ -94,7 +94,9 @@ module Datadog
                 {
                   id: content.path.config_id,
                   version: content.version,
-                  product: content.path.product
+                  product: content.path.product,
+                  apply_state: content.apply_state,
+                  apply_error: content.apply_error,
                 }
               end
             end

data/lib/datadog/core/remote/ext.rb

@@ -5,7 +5,7 @@ module Datadog
     module Remote
       module Ext
         ENV_ENABLED = 'DD_REMOTE_CONFIGURATION_ENABLED'
-        ENV_POLL_INTERVAL_SECONDS = 'DD_REMOTE_CONFIGURATION_POLL_INTERVAL_SECONDS'
+        ENV_POLL_INTERVAL_SECONDS = 'DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS'
       end
     end
   end

data/lib/datadog/core/remote/negotiation.rb

@@ -12,31 +12,44 @@ module Datadog
           transport_options[:agent_settings] = agent_settings if agent_settings
 
           @transport_root = Datadog::Core::Transport::HTTP.root(**transport_options.dup)
+          @logged = {}
         end
 
         def endpoint?(path)
           res = @transport_root.send_info
 
           if res.internal_error? && network_error?(res.error)
-            Datadog.logger.error { "agent unreachable: cannot negotiate #{path}" }
+            unless @logged[:agent_unreachable]
+              Datadog.logger.error { "agent unreachable: cannot negotiate #{path}" }
+              @logged[:agent_unreachable] = true
+            end
 
             return false
           end
 
           if res.not_found?
-            Datadog.logger.error { "agent reachable but has no /info endpoint: cannot negotiate #{path}" }
+            unless @logged[:no_info_endpoint]
+              Datadog.logger.error { "agent reachable but has no /info endpoint: cannot negotiate #{path}" }
+              @logged[:no_info_endpoint] = true
+            end
 
             return false
           end
 
           unless res.ok?
-            Datadog.logger.error { "agent reachable but unexpected response: cannot negotiate #{path}" }
+            unless @logged[:unexpected_response]
+              Datadog.logger.error { "agent reachable but unexpected response: cannot negotiate #{path}" }
+              @logged[:unexpected_response] = true
+            end
 
             return false
           end
 
           unless res.endpoints.include?(path)
-            Datadog.logger.error { "agent reachable but does not report #{path}" }
+            unless @logged[:no_config_endpoint]
+              Datadog.logger.error { "agent reachable but does not report #{path}" }
+              @logged[:no_config_endpoint] = true
+            end
 
             return false
           end

data/lib/datadog/core/runtime/ext.rb

@@ -1,26 +1,36 @@
+# frozen_string_literal: true
+
 module Datadog
   module Core
     module Runtime
       # @public_api
       module Ext
-        TAG_ID = 'runtime-id'.freeze
-        TAG_LANG = 'language'.freeze
-        TAG_PROCESS_ID = 'process_id'.freeze
+        TAG_ID = 'runtime-id'
+        TAG_LANG = 'language'
+        TAG_PROCESS_ID = 'process_id'
 
         # Metrics
         # @public_api
         module Metrics
-          ENV_ENABLED = 'DD_RUNTIME_METRICS_ENABLED'.freeze
+          ENV_ENABLED = 'DD_RUNTIME_METRICS_ENABLED'
 
-          METRIC_CLASS_COUNT = 'runtime.ruby.class_count'.freeze
-          METRIC_GC_PREFIX = 'runtime.ruby.gc'.freeze
-          METRIC_THREAD_COUNT = 'runtime.ruby.thread_count'.freeze
-          METRIC_GLOBAL_CONSTANT_STATE = 'runtime.ruby.global_constant_state'.freeze
-          METRIC_GLOBAL_METHOD_STATE = 'runtime.ruby.global_method_state'.freeze
-          METRIC_CONSTANT_CACHE_INVALIDATIONS = 'runtime.ruby.constant_cache_invalidations'.freeze
-          METRIC_CONSTANT_CACHE_MISSES = 'runtime.ruby.constant_cache_misses'.freeze
+          METRIC_CLASS_COUNT = 'runtime.ruby.class_count'
+          METRIC_GC_PREFIX = 'runtime.ruby.gc'
+          METRIC_THREAD_COUNT = 'runtime.ruby.thread_count'
+          METRIC_GLOBAL_CONSTANT_STATE = 'runtime.ruby.global_constant_state'
+          METRIC_GLOBAL_METHOD_STATE = 'runtime.ruby.global_method_state'
+          METRIC_CONSTANT_CACHE_INVALIDATIONS = 'runtime.ruby.constant_cache_invalidations'
+          METRIC_CONSTANT_CACHE_MISSES = 'runtime.ruby.constant_cache_misses'
+          METRIC_YJIT_CODE_GC_COUNT = 'runtime.ruby.yjit.code_gc_count'
+          METRIC_YJIT_CODE_REGION_SIZE = 'runtime.ruby.yjit.code_region_size'
+          METRIC_YJIT_FREED_CODE_SIZE = 'runtime.ruby.yjit.freed_code_size'
+          METRIC_YJIT_FREED_PAGE_COUNT = 'runtime.ruby.yjit.freed_page_count'
+          METRIC_YJIT_INLINE_CODE_SIZE = 'runtime.ruby.yjit.inline_code_size'
+          METRIC_YJIT_LIVE_PAGE_COUNT = 'runtime.ruby.yjit.live_page_count'
+          METRIC_YJIT_OBJECT_SHAPE_COUNT = 'runtime.ruby.yjit.object_shape_count'
+          METRIC_YJIT_OUTLINED_CODE_SIZE = 'runtime.ruby.yjit.outlined_code_size'
 
-          TAG_SERVICE = 'service'.freeze
+          TAG_SERVICE = 'service'
         end
       end
     end

data/lib/datadog/core/runtime/metrics.rb

@@ -5,6 +5,7 @@ require_relative '../environment/class_count'
 require_relative '../environment/gc'
 require_relative '../environment/thread_count'
 require_relative '../environment/vm_cache'
+require_relative '../environment/yjit'
 
 module Datadog
   module Core
@@ -78,6 +79,8 @@ module Datadog
               )
             end
           end
+
+          flush_yjit_stats
         end
 
         def gc_metrics
@@ -134,6 +137,46 @@ module Datadog
         def gauge_if_not_nil(metric_name, metric_value)
           gauge(metric_name, metric_value) if metric_value
         end
+
+        def flush_yjit_stats
+          # Only on Ruby >= 3.2
+          try_flush do
+            if Core::Environment::YJIT.available?
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_YJIT_CODE_GC_COUNT,
+                Core::Environment::YJIT.code_gc_count
+              )
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_YJIT_CODE_REGION_SIZE,
+                Core::Environment::YJIT.code_region_size
+              )
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_YJIT_FREED_CODE_SIZE,
+                Core::Environment::YJIT.freed_code_size
+              )
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_YJIT_FREED_PAGE_COUNT,
+                Core::Environment::YJIT.freed_page_count
+              )
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_YJIT_INLINE_CODE_SIZE,
+                Core::Environment::YJIT.inline_code_size
+              )
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_YJIT_LIVE_PAGE_COUNT,
+                Core::Environment::YJIT.live_page_count
+              )
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_YJIT_OBJECT_SHAPE_COUNT,
+                Core::Environment::YJIT.object_shape_count
+              )
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_YJIT_OUTLINED_CODE_SIZE,
+                Core::Environment::YJIT.outlined_code_size
+              )
+            end
+          end
+        end
       end
     end
   end

data/lib/datadog/core/telemetry/client.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'emitter'
 require_relative 'heartbeat'
 require_relative '../utils/forking'
@@ -16,12 +18,13 @@ module Datadog
         include Core::Utils::Forking
 
         # @param enabled [Boolean] Determines whether telemetry events should be sent to the API
-        def initialize(enabled: true)
+        # @param heartbeat_interval_seconds [Float] How frequently heartbeats will be reported, in seconds.
+        def initialize(heartbeat_interval_seconds:, enabled: true)
           @enabled = enabled
           @emitter = Emitter.new
           @stopped = false
           @unsupported = false
-          @worker = Telemetry::Heartbeat.new(enabled: @enabled) do
+          @worker = Telemetry::Heartbeat.new(enabled: @enabled, heartbeat_interval_seconds: heartbeat_interval_seconds) do
             heartbeat!
           end
         end
@@ -64,6 +67,13 @@ module Datadog
           @emitter.request(:'app-integrations-change')
         end
 
+        # Report configuration changes caused by Remote Configuration.
+        def client_configuration_change!(changes)
+          return if !@enabled || forked?
+
+          @emitter.request('app-client-configuration-change', data: { changes: changes, origin: 'remote_config' })
+        end
+
         private
 
         def heartbeat!

data/lib/datadog/core/telemetry/emitter.rb

@@ -21,11 +21,13 @@ module Datadog
 
         # Retrieves and emits a TelemetryRequest object based on the request type specified
         # @param request_type [String] the type of telemetry request to collect data for
-        def request(request_type)
+        # @param data [Object] arbitrary object to be passed to the respective `request_type` handler
+        def request(request_type, data: nil)
           begin
             request = Datadog::Core::Telemetry::Event.new.telemetry_request(
               request_type: request_type,
-              seq_id: self.class.sequence.next
+              seq_id: self.class.sequence.next,
+              data: data,
             ).to_h
             @http_transport.request(request_type: request_type.to_s, payload: request.to_json)
           rescue StandardError => e

data/lib/datadog/core/telemetry/event.rb

@@ -1,6 +1,9 @@
+# frozen_string_literal: true
+
 require_relative 'collector'
 require_relative 'v1/app_event'
 require_relative 'v1/telemetry_request'
+require_relative 'v2/app_client_configuration_change'
 
 module Datadog
   module Core
@@ -9,7 +12,7 @@ module Datadog
       class Event
         include Telemetry::Collector
 
-        API_VERSION = 'v1'.freeze
+        API_VERSION = 'v1'
 
         attr_reader \
           :api_version
@@ -21,12 +24,13 @@ module Datadog
         # Forms a TelemetryRequest object based on the event request_type
         # @param request_type [String] the type of telemetry request to collect data for
         # @param seq_id [Integer] the ID of the request; incremented each time a telemetry request is sent to the API
-        def telemetry_request(request_type:, seq_id:)
+        # @param data [Object] arbitrary object to be passed to the respective `request_type` handler
+        def telemetry_request(request_type:, seq_id:, data: nil)
           Telemetry::V1::TelemetryRequest.new(
             api_version: @api_version,
             application: application,
             host: host,
-            payload: payload(request_type),
+            payload: payload(request_type, data),
             request_type: request_type,
             runtime_id: runtime_id,
             seq_id: seq_id,
@@ -36,7 +40,7 @@ module Datadog
 
         private
 
-        def payload(request_type)
+        def payload(request_type, data)
           case request_type
           when :'app-started'
             app_started
@@ -44,6 +48,8 @@ module Datadog
             {}
           when :'app-integrations-change'
             app_integrations_change
+          when 'app-client-configuration-change'
+            app_client_configuration_change(data)
           else
             raise ArgumentError, "Request type invalid, received request_type: #{@request_type}"
           end
@@ -61,6 +67,15 @@ module Datadog
         def app_integrations_change
           Telemetry::V1::AppEvent.new(integrations: integrations)
         end
+
+        # DEV: During the transition from V1 to V2, the backend accepts many V2
+        # DEV: payloads through the V1 transport protocol.
+        # DEV: The `app-client-configuration-change` payload is one of them.
+        # DEV: Once V2 is fully implemented, `Telemetry::V2::AppClientConfigurationChange`
+        # DEV: should be reusable without major modifications.
+        def app_client_configuration_change(data)
+          Telemetry::V2::AppClientConfigurationChange.new(data[:changes], origin: data[:origin])
+        end
       end
     end
   end

data/lib/datadog/core/telemetry/ext.rb

@@ -1,8 +1,11 @@
+# frozen_string_literal: true
+
 module Datadog
   module Core
     module Telemetry
       module Ext
-        ENV_ENABLED = 'DD_INSTRUMENTATION_TELEMETRY_ENABLED'.freeze
+        ENV_ENABLED = 'DD_INSTRUMENTATION_TELEMETRY_ENABLED'
+        ENV_HEARTBEAT_INTERVAL = 'DD_TELEMETRY_HEARTBEAT_INTERVAL'
       end
     end
   end

data/lib/datadog/core/telemetry/heartbeat.rb

@@ -10,13 +10,11 @@ module Datadog
       class Heartbeat < Core::Worker
         include Core::Workers::Polling
 
-        DEFAULT_INTERVAL_SECONDS = 60
-
-        def initialize(enabled: true, interval: DEFAULT_INTERVAL_SECONDS, &block)
+        def initialize(heartbeat_interval_seconds:, enabled: true, &block)
           # Workers::Polling settings
           self.enabled = enabled
           # Workers::IntervalLoop settings
-          self.loop_base_interval = interval
+          self.loop_base_interval = heartbeat_interval_seconds
           self.fork_policy = Core::Workers::Async::Thread::FORK_POLICY_STOP
           super(&block)
           start

data/lib/datadog/core/telemetry/http/ext.rb

@@ -1,18 +1,20 @@
+# frozen_string_literal: true
+
 module Datadog
   module Core
     module Telemetry
       module Http
         module Ext
-          HEADER_DD_API_KEY = 'DD-API-KEY'.freeze
-          HEADER_CONTENT_TYPE = 'Content-Type'.freeze
-          HEADER_CONTENT_LENGTH = 'Content-Length'.freeze
-          HEADER_DD_TELEMETRY_API_VERSION = 'DD-Telemetry-API-Version'.freeze
-          HEADER_DD_TELEMETRY_REQUEST_TYPE = 'DD-Telemetry-Request-Type'.freeze
+          HEADER_DD_API_KEY = 'DD-API-KEY'
+          HEADER_CONTENT_TYPE = 'Content-Type'
+          HEADER_CONTENT_LENGTH = 'Content-Length'
+          HEADER_DD_TELEMETRY_API_VERSION = 'DD-Telemetry-API-Version'
+          HEADER_DD_TELEMETRY_REQUEST_TYPE = 'DD-Telemetry-Request-Type'
 
-          CONTENT_TYPE_APPLICATION_JSON = 'application/json'.freeze
-          API_VERSION = 'v1'.freeze
+          CONTENT_TYPE_APPLICATION_JSON = 'application/json'
+          API_VERSION = 'v1'
 
-          AGENT_ENDPOINT = '/telemetry/proxy/api/v2/apmtelemetry'.freeze
+          AGENT_ENDPOINT = '/telemetry/proxy/api/v2/apmtelemetry'
         end
       end
     end

data/lib/datadog/core/telemetry/http/transport.rb

@@ -38,6 +38,7 @@ module Datadog
 
           def headers(request_type:, api_version: Http::Ext::API_VERSION)
             {
+              Datadog::Transport::Ext::HTTP::HEADER_DD_INTERNAL_UNTRACED_REQUEST => '1',
               Http::Ext::HEADER_CONTENT_TYPE => Http::Ext::CONTENT_TYPE_APPLICATION_JSON,
               Http::Ext::HEADER_DD_TELEMETRY_API_VERSION => api_version,
               Http::Ext::HEADER_DD_TELEMETRY_REQUEST_TYPE => request_type,

data/lib/datadog/core/telemetry/v2/app_client_configuration_change.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require_relative 'request'
+
+module Datadog
+  module Core
+    module Telemetry
+      module V2
+        # Telemetry 'app-client-configuration-change' event.
+        # This request should contain client library configuration that have changes since the app-started event.
+        class AppClientConfigurationChange < Request
+          def initialize(configuration_changes, origin: 'unknown')
+            super('app-client-configuration-change')
+
+            @configuration_changes = configuration_changes
+            @origin = origin
+          end
+
+          # @see [Request#to_h]
+          def to_h
+            super.merge(payload: payload)
+          end
+
+          private
+
+          def payload
+            {
+              configuration: @configuration_changes.map do |name, value|
+                {
+                  name: name,
+                  value: value,
+                  origin: @origin,
+                }
+              end
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/v2/request.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Telemetry
+      module V2
+        # Base request object for Telemetry V2.
+        #
+        # `#to_h` is the main API, which returns a Ruby
+        # Hash that will be serialized as JSON.
+        class Request
+          # @param [String] request_type the Telemetry request type, which dictates how the Hash payload should be processed
+          def initialize(request_type)
+            @request_type = request_type
+          end
+
+          # Converts this request to a Hash that will
+          # be serialized as JSON.
+          # @return [Hash]
+          def to_h
+            {
+              request_type: @request_type
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/transport/http/client.rb

@@ -30,7 +30,7 @@ module Datadog
               "Internal error during #{self.class.name} request. Cause: #{e.class.name} #{e.message} " \
               "Location: #{Array(e.backtrace).first}"
 
-            Datadog.logger.error(message)
+            Datadog.logger.debug(message)
 
             Datadog::Transport::InternalErrorResponse.new(e)
           end

data/lib/datadog/core/transport/http/config.rb

@@ -126,6 +126,16 @@ module Datadog
               end.freeze
             end
 
+            def inspect
+              "#{super}, #{
+                {
+                  roots: @roots,
+                  targets: @targets,
+                  target_files: @target_files,
+                  client_configs: @client_configs,
+                }}"
+            end
+
             # When an expected key is missing
             class KeyError < StandardError
               def initialize(key)

data/lib/datadog/core/utils/duration.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Utils
+      # Helper methods for parsing string values into Numeric
+      module Duration
+        def self.call(value, base: :s)
+          cast = if value.include?('.')
+                   method(:Float)
+                 else
+                   method(:Integer)
+                 end
+
+          scale = case base
+                  when :s
+                    1_000_000_000
+                  when :ms
+                    1_000_000
+                  when :us
+                    1000
+                  when :ns
+                    1
+                  else
+                    raise ArgumentError, "invalid base: #{base.inspect}"
+                  end
+
+          result = case value
+                   when /^(\d+(?:\.\d+)?)h$/
+                     cast.call(Regexp.last_match(1)) * 1_000_000_000 * 60 * 60 / scale
+                   when /^(\d+(?:\.\d+)?)m$/
+                     cast.call(Regexp.last_match(1)) * 1_000_000_000 * 60 / scale
+                   when /^(\d+(?:\.\d+)?)s$/
+                     cast.call(Regexp.last_match(1)) * 1_000_000_000 / scale
+                   when /^(\d+(?:\.\d+)?)ms$/
+                     cast.call(Regexp.last_match(1)) * 1_000_000 / scale
+                   when /^(\d+(?:\.\d+)?)us$/
+                     cast.call(Regexp.last_match(1)) * 1_000 / scale
+                   when /^(\d+(?:\.\d+)?)ns$/
+                     cast.call(Regexp.last_match(1)) / scale
+                   when /^(\d+(?:\.\d+)?)$/
+                     cast.call(Regexp.last_match(1))
+                   else
+                     raise ArgumentError, "invalid duration: #{value.inspect}"
+                   end
+          # @type var result: Numeric
+          result.round
+        end
+      end
+    end
+  end
+end