cw-datadog 2.23.0.2

data/lib/datadog/core/remote/worker.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Remote
+      # Worker executes a block every interval on a separate Thread
+      class Worker
+        def initialize(interval:, logger:, &block)
+          @mutex = Mutex.new
+          @thr = nil
+
+          @starting = false
+          @started = false
+          @stopped = false
+
+          @interval = interval
+          @logger = logger
+          raise ArgumentError, 'can not initialize a worker without a block' unless block
+
+          @block = block
+        end
+
+        attr_reader :logger
+
+        def start
+          logger.debug { "remote worker starting (pid: #{Process.pid})" }
+
+          acquire_lock
+
+          if @stopped
+            logger.debug('remote worker: refusing to restart after previous stop')
+            return
+          end
+
+          return if @starting || @started
+
+          @starting = true
+
+          thread = Thread.new { poll(@interval) }
+          thread.name = self.class.name
+          thread.thread_variable_set(:fork_safe, true)
+          @thr = thread
+
+          @started = true
+          @starting = false
+
+          logger.debug { 'remote worker started' }
+        ensure
+          release_lock
+        end
+
+        def stop
+          logger.debug { "remote worker stopping (pid: #{Process.pid})" }
+
+          acquire_lock
+
+          thread = @thr
+
+          if thread
+            thread.kill
+            thread.join
+          end
+
+          @started = false
+          @thr = nil
+          @stopped = true
+
+          logger.debug { 'remote worker stopped' }
+        ensure
+          release_lock
+        end
+
+        def started?
+          @started
+        end
+
+        private
+
+        def acquire_lock
+          @mutex.lock
+        end
+
+        def release_lock
+          @mutex.unlock
+        end
+
+        def poll(interval)
+          loop do
+            break unless @mutex.synchronize { @starting || @started }
+
+            call
+
+            sleep(interval)
+          end
+        end
+
+        def call
+          logger.debug { 'remote worker perform' }
+
+          @block.call
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/remote.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    # Module to interact with core remote component
+    module Remote
+      class << self
+        def active_remote
+          remote
+        end
+
+        private
+
+        def components
+          Datadog.send(:components)
+        end
+
+        def remote
+          components.remote
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/runtime/ext.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Runtime
+      # @public_api
+      module Ext
+        TAG_ID = 'runtime-id'
+        TAG_LANG = 'language'
+        TAG_PROCESS_ID = 'process_id'
+
+        # Metrics
+        # @public_api
+        module Metrics
+          ENV_ENABLED = 'DD_RUNTIME_METRICS_ENABLED'
+
+          METRIC_CLASS_COUNT = 'runtime.ruby.class_count'
+          METRIC_GC_PREFIX = 'runtime.ruby.gc'
+          METRIC_THREAD_COUNT = 'runtime.ruby.thread_count'
+          METRIC_GLOBAL_CONSTANT_STATE = 'runtime.ruby.global_constant_state'
+          METRIC_GLOBAL_METHOD_STATE = 'runtime.ruby.global_method_state'
+          METRIC_CONSTANT_CACHE_INVALIDATIONS = 'runtime.ruby.constant_cache_invalidations'
+          METRIC_CONSTANT_CACHE_MISSES = 'runtime.ruby.constant_cache_misses'
+          METRIC_YJIT_CODE_GC_COUNT = 'runtime.ruby.yjit.code_gc_count'
+          METRIC_YJIT_CODE_REGION_SIZE = 'runtime.ruby.yjit.code_region_size'
+          METRIC_YJIT_FREED_CODE_SIZE = 'runtime.ruby.yjit.freed_code_size'
+          METRIC_YJIT_FREED_PAGE_COUNT = 'runtime.ruby.yjit.freed_page_count'
+          METRIC_YJIT_INLINE_CODE_SIZE = 'runtime.ruby.yjit.inline_code_size'
+          METRIC_YJIT_LIVE_PAGE_COUNT = 'runtime.ruby.yjit.live_page_count'
+          METRIC_YJIT_OBJECT_SHAPE_COUNT = 'runtime.ruby.yjit.object_shape_count'
+          METRIC_YJIT_OUTLINED_CODE_SIZE = 'runtime.ruby.yjit.outlined_code_size'
+          METRIC_YJIT_YJIT_ALLOC_SIZE = 'runtime.ruby.yjit.yjit_alloc_size'
+          METRIC_YJIT_RATIO_IN_YJIT = 'runtime.ruby.yjit.ratio_in_yjit'
+
+          TAG_SERVICE = 'service'
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/runtime/metrics.rb

@@ -0,0 +1,202 @@
+# frozen_string_literal: true
+
+require_relative 'ext'
+
+require_relative '../metrics/client'
+require_relative '../environment/class_count'
+require_relative '../environment/gc'
+require_relative '../environment/thread_count'
+require_relative '../environment/vm_cache'
+require_relative '../environment/yjit'
+
+module Datadog
+  module Core
+    module Runtime
+      # For generating runtime metrics
+      class Metrics < Core::Metrics::Client
+        def initialize(telemetry:, **options)
+          super
+
+          # Initialize service list
+          @services = Set.new(options.fetch(:services, []))
+          @service_tags = nil
+          compile_service_tags!
+
+          # Initialize the collection of runtime-id
+          @runtime_id_enabled = options.fetch(:experimental_runtime_id_enabled, false)
+        end
+
+        # Associate service with runtime metrics
+        def register_service(service)
+          return unless enabled? && service
+
+          service = service.to_s
+
+          unless @services.include?(service)
+            # Add service to list and update services tag
+            services << service
+
+            # Recompile the service tags
+            compile_service_tags!
+          end
+        end
+
+        # Flush all runtime metrics to Statsd client
+        def flush
+          return unless enabled?
+
+          try_flush do
+            if Core::Environment::ClassCount.available?
+              gauge(Core::Runtime::Ext::Metrics::METRIC_CLASS_COUNT, Core::Environment::ClassCount.value)
+            end
+          end
+
+          try_flush do
+            if Core::Environment::ThreadCount.available?
+              gauge(Core::Runtime::Ext::Metrics::METRIC_THREAD_COUNT, Core::Environment::ThreadCount.value)
+            end
+          end
+
+          try_flush { gc_metrics.each { |metric, value| gauge(metric, value) } if Core::Environment::GC.available? }
+
+          try_flush do
+            if Core::Environment::VMCache.available?
+              # Only on Ruby < 3.2
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_GLOBAL_CONSTANT_STATE,
+                Core::Environment::VMCache.global_constant_state
+              )
+
+              # Only on Ruby 2.x
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_GLOBAL_METHOD_STATE,
+                Core::Environment::VMCache.global_method_state
+              )
+
+              # Only on Ruby >= 3.2
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_CONSTANT_CACHE_INVALIDATIONS,
+                Core::Environment::VMCache.constant_cache_invalidations
+              )
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_CONSTANT_CACHE_MISSES,
+                Core::Environment::VMCache.constant_cache_misses
+              )
+            end
+          end
+
+          flush_yjit_stats
+        end
+
+        def gc_metrics
+          Core::Environment::GC.stat.flat_map do |k, v|
+            nested_gc_metric(Core::Runtime::Ext::Metrics::METRIC_GC_PREFIX, k, v)
+          end.to_h
+        end
+
+        def try_flush
+          yield
+        rescue => e
+          Datadog.logger.warn("Error while sending runtime metric. Cause: #{e.class.name} #{e.message}")
+        end
+
+        def default_metric_options
+          # Return dupes, so that the constant isn't modified,
+          # and defaults are unfrozen for mutation in Statsd.
+          super.tap do |options|
+            options[:tags] = options[:tags].dup
+
+            # Add services dynamically because they might change during runtime.
+            options[:tags].concat(service_tags) unless service_tags.nil?
+
+            # Add runtime-id dynamically because it might change during runtime.
+            options[:tags].concat(["runtime-id:#{Core::Environment::Identity.id}"]) if @runtime_id_enabled
+          end
+        end
+
+        private
+
+        attr_reader \
+          :service_tags,
+          :services,
+          :runtime_id_enabled
+
+        def compile_service_tags!
+          @service_tags = services.to_a.collect do |service|
+            "#{Core::Runtime::Ext::Metrics::TAG_SERVICE}:#{service}"
+          end
+        end
+
+        def nested_gc_metric(prefix, k, v)
+          path = "#{prefix}.#{k}"
+
+          if v.is_a?(Hash)
+            v.flat_map do |key, value|
+              nested_gc_metric(path, key, value)
+            end
+          else
+            [[to_metric_name(path), v]]
+          end
+        end
+
+        def to_metric_name(str)
+          str.downcase.gsub(/[-\s]/, '_')
+        end
+
+        def gauge_if_not_nil(metric_name, metric_value)
+          gauge(metric_name, metric_value) if metric_value
+        end
+
+        # standard:disable Metrics/MethodLength
+        def flush_yjit_stats
+          # Only on Ruby >= 3.2
+          try_flush do
+            if Core::Environment::YJIT.available?
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_YJIT_CODE_GC_COUNT,
+                Core::Environment::YJIT.code_gc_count
+              )
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_YJIT_CODE_REGION_SIZE,
+                Core::Environment::YJIT.code_region_size
+              )
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_YJIT_FREED_CODE_SIZE,
+                Core::Environment::YJIT.freed_code_size
+              )
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_YJIT_FREED_PAGE_COUNT,
+                Core::Environment::YJIT.freed_page_count
+              )
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_YJIT_INLINE_CODE_SIZE,
+                Core::Environment::YJIT.inline_code_size
+              )
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_YJIT_LIVE_PAGE_COUNT,
+                Core::Environment::YJIT.live_page_count
+              )
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_YJIT_OBJECT_SHAPE_COUNT,
+                Core::Environment::YJIT.object_shape_count
+              )
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_YJIT_OUTLINED_CODE_SIZE,
+                Core::Environment::YJIT.outlined_code_size
+              )
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_YJIT_YJIT_ALLOC_SIZE,
+                Core::Environment::YJIT.yjit_alloc_size
+              )
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_YJIT_RATIO_IN_YJIT,
+                Core::Environment::YJIT.ratio_in_yjit
+              )
+            end
+          end
+        end
+        # standard:enable Metrics/MethodLength
+      end
+    end
+  end
+end

data/lib/datadog/core/semaphore.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    # Semaphore pattern implementation, as described in documentation for
+    # ConditionVariable.
+    #
+    # @api private
+    class Semaphore
+      def initialize
+        @wake_lock = Mutex.new
+        @wake = ConditionVariable.new
+      end
+
+      def signal
+        wake_lock.synchronize do
+          wake.signal
+        end
+      end
+
+      def wait(timeout = nil)
+        wake_lock.synchronize do
+          # steep specifies that the second argument to wait is of type
+          # ::Time::_Timeout which for some reason is not Numeric and is not
+          # castable from Numeric.
+          wake.wait(wake_lock, timeout) # steep:ignore
+        end
+      end
+
+      private
+
+      attr_reader :wake_lock, :wake
+    end
+  end
+end

data/lib/datadog/core/tag_builder.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require_relative 'environment/socket'
+require_relative 'environment/identity'
+require_relative 'environment/git'
+
+module Datadog
+  module Core
+    # This module builds a hash of tags.
+    #
+    # @api private
+    module TagBuilder
+      def self.fixed_environment_tags
+        @fixed_environment_tags ||= {
+          'language' => Environment::Identity.lang,
+          'runtime' => Environment::Identity.lang, # Known to be duplicated from language above
+          'runtime_engine' => Environment::Identity.lang_engine,
+          'runtime_platform' => Environment::Identity.lang_platform,
+          'runtime_version' => Environment::Identity.lang_version,
+          'library_version' => Environment::Identity.gem_datadog_version,
+          'git.repository_url' => Environment::Git.git_repository_url,
+          'git.commit.sha' => Environment::Git.git_commit_sha,
+        }.compact.freeze
+      end
+
+      def self.tags(settings)
+        # Note that user tags get overwritten by our tags, and also
+        # that user tags do not get compacted (nil values are sent as
+        # empty strings).
+        settings.tags.merge(fixed_environment_tags).merge({
+          # Hostname can possibly change during application runtime.
+          'host' => Environment::Socket.hostname,
+          # Runtime ID changes upon a fork.
+          'runtime-id' => Environment::Identity.id,
+          # Process ID changes upon a fork.
+          'process_id' => Process.pid.to_s,
+          # Unified service tagging.
+          'env' => settings.env,
+          'service' => settings.service,
+          'version' => settings.version,
+        }.compact)
+      end
+
+      def self.serialize_tags(tags)
+        # DEV: Should there be some sort of escaping done here?
+        tags.map do |key, value|
+          "#{key}:#{value}"
+        end.join(',')
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/component.rb

@@ -0,0 +1,206 @@
+# frozen_string_literal: true
+
+require_relative 'emitter'
+require_relative 'event'
+require_relative 'metrics_manager'
+require_relative 'worker'
+require_relative 'logging'
+require_relative 'transport/http'
+
+require_relative '../configuration/ext'
+require_relative '../configuration/agentless_settings_resolver'
+require_relative '../utils/forking'
+
+module Datadog
+  module Core
+    module Telemetry
+      # Telemetry entrypoint, coordinates sending telemetry events at various points in app lifecycle.
+      # Note: Telemetry does not spawn its worker thread in fork processes, thus no telemetry is sent in forked processes.
+      #
+      # @api private
+      class Component
+        ENDPOINT_COLLECTION_MESSAGE_LIMIT = 300
+
+        attr_reader :enabled, :logger, :transport, :worker
+
+        include Core::Utils::Forking
+        include Telemetry::Logging
+
+        def self.build(settings, agent_settings, logger)
+          enabled = settings.telemetry.enabled
+          agentless_enabled = settings.telemetry.agentless_enabled
+
+          if agentless_enabled && settings.api_key.nil?
+            enabled = false
+            logger.debug { 'Telemetry disabled. Agentless telemetry requires a DD_API_KEY variable to be set.' }
+          end
+
+          Telemetry::Component.new(
+            settings: settings,
+            agent_settings: agent_settings,
+            enabled: enabled,
+            logger: logger,
+          )
+        end
+
+        # @param enabled [Boolean] Determines whether telemetry events should be sent to the API
+        def initialize( # standard:disable Metrics/MethodLength
+          settings:,
+          agent_settings:,
+          logger:,
+          enabled:
+        )
+          @enabled = enabled
+          @log_collection_enabled = settings.telemetry.log_collection_enabled
+          @logger = logger
+
+          @metrics_manager = MetricsManager.new(
+            enabled: @enabled && settings.telemetry.metrics_enabled,
+            aggregation_interval: settings.telemetry.metrics_aggregation_interval_seconds,
+          )
+
+          @stopped = false
+
+          return unless @enabled
+
+          @transport = if settings.telemetry.agentless_enabled
+            # We don't touch the `agent_settings` since we still want the telemetry payloads to refer to the original
+            # settings, even though the telemetry itself may be using a different path.
+            telemetry_specific_agent_settings = Core::Configuration::AgentlessSettingsResolver.call(
+              settings,
+              host_prefix: 'instrumentation-telemetry-intake',
+              url_override: settings.telemetry.agentless_url_override,
+              url_override_source: 'c.telemetry.agentless_url_override',
+              logger: logger,
+            )
+            Telemetry::Transport::HTTP.agentless_telemetry(
+              agent_settings: telemetry_specific_agent_settings,
+              logger: logger,
+              # api_key should have already validated to be
+              # not nil by +build+ method above.
+              api_key: settings.api_key,
+            )
+          else
+            Telemetry::Transport::HTTP.agent_telemetry(
+              agent_settings: agent_settings, logger: logger,
+            )
+          end
+
+          @worker = Telemetry::Worker.new(
+            enabled: @enabled,
+            heartbeat_interval_seconds: settings.telemetry.heartbeat_interval_seconds,
+            metrics_aggregation_interval_seconds: settings.telemetry.metrics_aggregation_interval_seconds,
+            emitter: Emitter.new(
+              @transport,
+              logger: @logger,
+              debug: settings.telemetry.debug,
+            ),
+            metrics_manager: @metrics_manager,
+            dependency_collection: settings.telemetry.dependency_collection,
+            logger: logger,
+            shutdown_timeout: settings.telemetry.shutdown_timeout_seconds,
+          )
+
+          @agent_settings = agent_settings
+        end
+
+        def disable!
+          @enabled = false
+          @worker&.enabled = false
+        end
+
+        def start(initial_event_is_change = false)
+          return if !@enabled
+
+          initial_event = if initial_event_is_change
+            Event::SynthAppClientConfigurationChange.new(agent_settings: @agent_settings)
+          else
+            Event::AppStarted.new(agent_settings: @agent_settings)
+          end
+
+          @worker.start(initial_event)
+        end
+
+        def shutdown!
+          return if @stopped
+
+          if defined?(@worker)
+            @worker&.stop(true)
+          end
+
+          @stopped = true
+        end
+
+        def emit_closing!
+          return if !@enabled || forked?
+
+          @worker.enqueue(Event::AppClosing.new)
+        end
+
+        def integrations_change!
+          return if !@enabled || forked?
+
+          @worker.enqueue(Event::AppIntegrationsChange.new)
+        end
+
+        def log!(event)
+          return if !@enabled || forked? || !@log_collection_enabled
+
+          @worker.enqueue(event)
+        end
+
+        # Wait for the worker to send out all events that have already
+        # been queued, up to 15 seconds. Returns whether all events have
+        # been flushed.
+        #
+        # @api private
+        def flush
+          return if !@enabled || forked?
+
+          @worker.flush
+        end
+
+        # Report configuration changes caused by Remote Configuration.
+        def client_configuration_change!(changes)
+          return if !@enabled || forked?
+
+          @worker.enqueue(Event::AppClientConfigurationChange.new(changes, 'remote_config'))
+        end
+
+        # Report application endpoints
+        def app_endpoints_loaded(endpoints, page_size: ENDPOINT_COLLECTION_MESSAGE_LIMIT)
+          return if !@enabled || forked?
+
+          endpoints.each_slice(page_size).with_index do |endpoints_slice, i|
+            @worker.enqueue(Event::AppEndpointsLoaded.new(endpoints_slice, is_first: i.zero?))
+          end
+        end
+
+        # Increments a count metric.
+        def inc(namespace, metric_name, value, tags: {}, common: true)
+          @metrics_manager.inc(namespace, metric_name, value, tags: tags, common: common)
+        end
+
+        # Decremenets a count metric.
+        def dec(namespace, metric_name, value, tags: {}, common: true)
+          @metrics_manager.dec(namespace, metric_name, value, tags: tags, common: common)
+        end
+
+        # Tracks gauge metric.
+        def gauge(namespace, metric_name, value, tags: {}, common: true)
+          @metrics_manager.gauge(namespace, metric_name, value, tags: tags, common: common)
+        end
+
+        # Tracks rate metric.
+        def rate(namespace, metric_name, value, tags: {}, common: true)
+          @metrics_manager.rate(namespace, metric_name, value, tags: tags, common: common)
+        end
+
+        # Tracks distribution metric.
+        def distribution(namespace, metric_name, value, tags: {}, common: true)
+          @metrics_manager.distribution(namespace, metric_name, value, tags: tags, common: common)
+        end
+      end
+    end
+  end
+end