datadog 2.0.0.beta1

data/lib/datadog/core/remote/worker.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Remote
+      # Worker executes a block every interval on a separate Thread
+      class Worker
+        def initialize(interval:, &block)
+          @mutex = Mutex.new
+          @thr = nil
+
+          @starting = false
+          @started = false
+          @stopped = false
+
+          @interval = interval
+          raise ArgumentError, 'can not initialize a worker without a block' unless block
+
+          @block = block
+        end
+
+        def start
+          Datadog.logger.debug { 'remote worker starting' }
+
+          acquire_lock
+
+          if @stopped
+            Datadog.logger.debug('remote worker: refusing to restart after previous stop')
+            return
+          end
+
+          return if @starting || @started
+
+          @starting = true
+
+          thread = Thread.new { poll(@interval) }
+          thread.name = self.class.name unless Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.3')
+          thread.thread_variable_set(:fork_safe, true)
+          @thr = thread
+
+          @started = true
+          @starting = false
+
+          Datadog.logger.debug { 'remote worker started' }
+        ensure
+          release_lock
+        end
+
+        def stop
+          Datadog.logger.debug { 'remote worker stopping' }
+
+          acquire_lock
+
+          thread = @thr
+
+          if thread
+            thread.kill
+            thread.join
+          end
+
+          @started = false
+          @thr = nil
+          @stopped = true
+
+          Datadog.logger.debug { 'remote worker stopped' }
+        ensure
+          release_lock
+        end
+
+        def started?
+          @started
+        end
+
+        private
+
+        def acquire_lock
+          @mutex.lock
+        end
+
+        def release_lock
+          @mutex.unlock
+        end
+
+        def poll(interval)
+          loop do
+            break unless @mutex.synchronize { @starting || @started }
+
+            call
+
+            sleep(interval)
+          end
+        end
+
+        def call
+          Datadog.logger.debug { 'remote worker perform' }
+
+          @block.call
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/remote.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    # Module to interact with core remote component
+    module Remote
+      class << self
+        def active_remote
+          remote
+        end
+
+        private
+
+        def components
+          Datadog.send(:components)
+        end
+
+        def remote
+          components.remote
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/runtime/ext.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Runtime
+      # @public_api
+      module Ext
+        TAG_ID = 'runtime-id'
+        TAG_LANG = 'language'
+        TAG_PROCESS_ID = 'process_id'
+
+        # Metrics
+        # @public_api
+        module Metrics
+          ENV_ENABLED = 'DD_RUNTIME_METRICS_ENABLED'
+
+          METRIC_CLASS_COUNT = 'runtime.ruby.class_count'
+          METRIC_GC_PREFIX = 'runtime.ruby.gc'
+          METRIC_THREAD_COUNT = 'runtime.ruby.thread_count'
+          METRIC_GLOBAL_CONSTANT_STATE = 'runtime.ruby.global_constant_state'
+          METRIC_GLOBAL_METHOD_STATE = 'runtime.ruby.global_method_state'
+          METRIC_CONSTANT_CACHE_INVALIDATIONS = 'runtime.ruby.constant_cache_invalidations'
+          METRIC_CONSTANT_CACHE_MISSES = 'runtime.ruby.constant_cache_misses'
+          METRIC_YJIT_CODE_GC_COUNT = 'runtime.ruby.yjit.code_gc_count'
+          METRIC_YJIT_CODE_REGION_SIZE = 'runtime.ruby.yjit.code_region_size'
+          METRIC_YJIT_FREED_CODE_SIZE = 'runtime.ruby.yjit.freed_code_size'
+          METRIC_YJIT_FREED_PAGE_COUNT = 'runtime.ruby.yjit.freed_page_count'
+          METRIC_YJIT_INLINE_CODE_SIZE = 'runtime.ruby.yjit.inline_code_size'
+          METRIC_YJIT_LIVE_PAGE_COUNT = 'runtime.ruby.yjit.live_page_count'
+          METRIC_YJIT_OBJECT_SHAPE_COUNT = 'runtime.ruby.yjit.object_shape_count'
+          METRIC_YJIT_OUTLINED_CODE_SIZE = 'runtime.ruby.yjit.outlined_code_size'
+
+          TAG_SERVICE = 'service'
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/runtime/metrics.rb

@@ -0,0 +1,185 @@
+# frozen_string_literal: true
+
+require_relative 'ext'
+
+require_relative '../metrics/client'
+require_relative '../environment/class_count'
+require_relative '../environment/gc'
+require_relative '../environment/thread_count'
+require_relative '../environment/vm_cache'
+require_relative '../environment/yjit'
+
+module Datadog
+  module Core
+    module Runtime
+      # For generating runtime metrics
+      class Metrics < Core::Metrics::Client
+        def initialize(**options)
+          super
+
+          # Initialize service list
+          @services = Set.new(options.fetch(:services, []))
+          @service_tags = nil
+          compile_service_tags!
+        end
+
+        # Associate service with runtime metrics
+        def register_service(service)
+          return unless enabled? && service
+
+          service = service.to_s
+
+          unless @services.include?(service)
+            # Add service to list and update services tag
+            services << service
+
+            # Recompile the service tags
+            compile_service_tags!
+          end
+        end
+
+        # Flush all runtime metrics to Statsd client
+        def flush
+          return unless enabled?
+
+          try_flush do
+            if Core::Environment::ClassCount.available?
+              gauge(Core::Runtime::Ext::Metrics::METRIC_CLASS_COUNT, Core::Environment::ClassCount.value)
+            end
+          end
+
+          try_flush do
+            if Core::Environment::ThreadCount.available?
+              gauge(Core::Runtime::Ext::Metrics::METRIC_THREAD_COUNT, Core::Environment::ThreadCount.value)
+            end
+          end
+
+          try_flush { gc_metrics.each { |metric, value| gauge(metric, value) } if Core::Environment::GC.available? }
+
+          try_flush do
+            if Core::Environment::VMCache.available?
+              # Only on Ruby < 3.2
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_GLOBAL_CONSTANT_STATE,
+                Core::Environment::VMCache.global_constant_state
+              )
+
+              # Only on Ruby 2.x
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_GLOBAL_METHOD_STATE,
+                Core::Environment::VMCache.global_method_state
+              )
+
+              # Only on Ruby >= 3.2
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_CONSTANT_CACHE_INVALIDATIONS,
+                Core::Environment::VMCache.constant_cache_invalidations
+              )
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_CONSTANT_CACHE_MISSES,
+                Core::Environment::VMCache.constant_cache_misses
+              )
+            end
+          end
+
+          flush_yjit_stats
+        end
+
+        def gc_metrics
+          Core::Environment::GC.stat.flat_map do |k, v|
+            nested_gc_metric(Core::Runtime::Ext::Metrics::METRIC_GC_PREFIX, k, v)
+          end.to_h
+        end
+
+        def try_flush
+          yield
+        rescue StandardError => e
+          Datadog.logger.error("Error while sending runtime metric. Cause: #{e.class.name} #{e.message}")
+        end
+
+        def default_metric_options
+          # Return dupes, so that the constant isn't modified,
+          # and defaults are unfrozen for mutation in Statsd.
+          super.tap do |options|
+            options[:tags] = options[:tags].dup
+
+            # Add services dynamically because they might change during runtime.
+            options[:tags].concat(service_tags) unless service_tags.nil?
+          end
+        end
+
+        private
+
+        attr_reader \
+          :service_tags,
+          :services
+
+        def compile_service_tags!
+          @service_tags = services.to_a.collect do |service|
+            "#{Core::Runtime::Ext::Metrics::TAG_SERVICE}:#{service}"
+          end
+        end
+
+        def nested_gc_metric(prefix, k, v)
+          path = "#{prefix}.#{k}"
+
+          if v.is_a?(Hash)
+            v.flat_map do |key, value|
+              nested_gc_metric(path, key, value)
+            end
+          else
+            [[to_metric_name(path), v]]
+          end
+        end
+
+        def to_metric_name(str)
+          str.downcase.gsub(/[-\s]/, '_')
+        end
+
+        def gauge_if_not_nil(metric_name, metric_value)
+          gauge(metric_name, metric_value) if metric_value
+        end
+
+        def flush_yjit_stats
+          # Only on Ruby >= 3.2
+          try_flush do
+            if Core::Environment::YJIT.available?
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_YJIT_CODE_GC_COUNT,
+                Core::Environment::YJIT.code_gc_count
+              )
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_YJIT_CODE_REGION_SIZE,
+                Core::Environment::YJIT.code_region_size
+              )
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_YJIT_FREED_CODE_SIZE,
+                Core::Environment::YJIT.freed_code_size
+              )
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_YJIT_FREED_PAGE_COUNT,
+                Core::Environment::YJIT.freed_page_count
+              )
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_YJIT_INLINE_CODE_SIZE,
+                Core::Environment::YJIT.inline_code_size
+              )
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_YJIT_LIVE_PAGE_COUNT,
+                Core::Environment::YJIT.live_page_count
+              )
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_YJIT_OBJECT_SHAPE_COUNT,
+                Core::Environment::YJIT.object_shape_count
+              )
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_YJIT_OUTLINED_CODE_SIZE,
+                Core::Environment::YJIT.outlined_code_size
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/client.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+require_relative 'emitter'
+require_relative 'heartbeat'
+require_relative '../utils/forking'
+
+module Datadog
+  module Core
+    module Telemetry
+      # Telemetry entrypoint, coordinates sending telemetry events at various points in app lifecycle.
+      class Client
+        attr_reader \
+          :emitter,
+          :enabled,
+          :unsupported,
+          :worker
+
+        include Core::Utils::Forking
+
+        # @param enabled [Boolean] Determines whether telemetry events should be sent to the API
+        # @param heartbeat_interval_seconds [Float] How frequently heartbeats will be reported, in seconds.
+        def initialize(heartbeat_interval_seconds:, enabled: true)
+          @enabled = enabled
+          @emitter = Emitter.new
+          @stopped = false
+          @unsupported = false
+          @worker = Telemetry::Heartbeat.new(enabled: @enabled, heartbeat_interval_seconds: heartbeat_interval_seconds) do
+            heartbeat!
+          end
+        end
+
+        def disable!
+          @enabled = false
+          @worker.enabled = false
+        end
+
+        def started!
+          return if !@enabled || forked?
+
+          res = @emitter.request(:'app-started')
+
+          if res.not_found? # Telemetry is only supported by agent versions 7.34 and up
+            Datadog.logger.debug('Agent does not support telemetry; disabling future telemetry events.')
+            disable!
+            @unsupported = true # Prevent telemetry from getting re-enabled
+          end
+
+          res
+        end
+
+        def emit_closing!
+          return if !@enabled || forked?
+
+          @emitter.request(:'app-closing')
+        end
+
+        def stop!
+          return if @stopped
+
+          @worker.stop(true, 0)
+          @stopped = true
+        end
+
+        def integrations_change!
+          return if !@enabled || forked?
+
+          @emitter.request(:'app-integrations-change')
+        end
+
+        # Report configuration changes caused by Remote Configuration.
+        def client_configuration_change!(changes)
+          return if !@enabled || forked?
+
+          @emitter.request('app-client-configuration-change', data: { changes: changes, origin: 'remote_config' })
+        end
+
+        private
+
+        def heartbeat!
+          return if !@enabled || forked?
+
+          @emitter.request(:'app-heartbeat')
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/collector.rb

@@ -0,0 +1,248 @@
+# frozen_string_literal: true
+
+require 'etc'
+
+require_relative '../configuration/agent_settings_resolver'
+require_relative '../environment/ext'
+require_relative '../environment/platform'
+require_relative '../utils/hash'
+require_relative 'v1/application'
+require_relative 'v1/dependency'
+require_relative 'v1/host'
+require_relative 'v1/install_signature'
+require_relative 'v1/integration'
+require_relative 'v1/product'
+require_relative '../transport/ext'
+
+module Datadog
+  module Core
+    module Telemetry
+      # Module defining methods for collecting metadata for telemetry
+      module Collector
+        include Datadog::Core::Configuration
+        using Core::Utils::Hash::Refinement
+
+        # Forms a hash of configuration key value pairs to be sent in the additional payload
+        def additional_payload
+          additional_payload_variables
+        end
+
+        # Forms a telemetry application object
+        def application
+          Telemetry::V1::Application.new(
+            env: env,
+            language_name: Datadog::Core::Environment::Ext::LANG,
+            language_version: Datadog::Core::Environment::Ext::LANG_VERSION,
+            products: products,
+            runtime_name: Datadog::Core::Environment::Ext::RUBY_ENGINE,
+            runtime_version: Datadog::Core::Environment::Ext::ENGINE_VERSION,
+            service_name: service_name,
+            service_version: service_version,
+            tracer_version: library_version
+          )
+        end
+
+        # Forms a hash of standard key value pairs to be sent in the app-started event configuration
+        def configurations
+          config = Datadog.configuration
+          hash = {
+            DD_AGENT_HOST: config.agent.host,
+            DD_AGENT_TRANSPORT: agent_transport,
+            DD_TRACE_SAMPLE_RATE: format_configuration_value(config.tracing.sampling.default_rate),
+            DD_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED: config.tracing.contrib.global_default_service_name.enabled
+          }
+          peer_service_mapping_str = ''
+          unless config.tracing.contrib.peer_service_mapping.empty?
+            peer_service_mapping = config.tracing.contrib.peer_service_mapping
+            peer_service_mapping_str = peer_service_mapping.map { |key, value| "#{key}:#{value}" }.join(',')
+          end
+          hash[:DD_TRACE_PEER_SERVICE_MAPPING] = peer_service_mapping_str
+          hash.compact!
+          hash
+        end
+
+        # Forms a telemetry app-started dependencies object
+        def dependencies
+          Gem.loaded_specs.collect do |name, loaded_gem|
+            Datadog::Core::Telemetry::V1::Dependency.new(
+              # `hash` should be used when `version` is not available
+              name: name, version: loaded_gem.version.to_s, hash: nil
+            )
+          end
+        end
+
+        # Forms a telemetry host object
+        def host
+          Telemetry::V1::Host.new(
+            container_id: Core::Environment::Container.container_id,
+            hostname: Core::Environment::Platform.hostname,
+            kernel_name: Core::Environment::Platform.kernel_name,
+            kernel_release: Core::Environment::Platform.kernel_release,
+            kernel_version: Core::Environment::Platform.kernel_version
+          )
+        end
+
+        # Forms a telemetry app-started install_signature object
+        def install_signature
+          Telemetry::V1::InstallSignature.new(
+            install_id: Datadog.configuration.dig('telemetry', 'install_id'),
+            install_type: Datadog.configuration.dig('telemetry', 'install_type'),
+            install_time: Datadog.configuration.dig('telemetry', 'install_time'),
+          )
+        end
+
+        # Forms a telemetry app-started integrations object
+        def integrations
+          Datadog.registry.map do |integration|
+            is_instrumented = instrumented?(integration)
+            is_enabled = is_instrumented && patched?(integration)
+            Telemetry::V1::Integration.new(
+              name: integration.name.to_s,
+              enabled: is_enabled,
+              version: integration_version(integration),
+              compatible: integration_compatible?(integration),
+              error: (patch_error(integration) if is_instrumented && !is_enabled),
+              auto_enabled: is_enabled ? integration_auto_instrument?(integration) : nil
+            )
+          end
+        end
+
+        # Returns the runtime ID of the current process
+        def runtime_id
+          Datadog::Core::Environment::Identity.id
+        end
+
+        # Returns the current as a UNIX timestamp in seconds
+        def tracer_time
+          Time.now.to_i
+        end
+
+        private
+
+        TARGET_OPTIONS = [
+          'logger.level',
+          'profiling.advanced.code_provenance_enabled',
+          'profiling.advanced.endpoint.collection.enabled',
+          'profiling.enabled',
+          'runtime_metrics.enabled',
+          'tracing.analytics.enabled',
+          'tracing.propagation_style_inject',
+          'tracing.propagation_style_extract',
+          'tracing.enabled',
+          'tracing.log_injection',
+          'tracing.partial_flush.enabled',
+          'tracing.partial_flush.min_spans_threshold',
+          'tracing.report_hostname',
+          'tracing.sampling.default_rate',
+          'tracing.sampling.rate_limit'
+        ].freeze
+
+        def additional_payload_variables
+          # Whitelist of configuration options to send in additional payload object
+          configuration = Datadog.configuration
+          options = TARGET_OPTIONS.each_with_object({}) do |option, hash|
+            split_option = option.split('.')
+            hash[option] = format_configuration_value(configuration.dig(*split_option))
+          end
+
+          # Add some more custom additional payload values here
+          options['tracing.auto_instrument.enabled'] = !defined?(Datadog::AutoInstrument::LOADED).nil?
+          options['tracing.writer_options.buffer_size'] =
+            format_configuration_value(configuration.tracing.writer_options[:buffer_size])
+          options['tracing.writer_options.flush_interval'] =
+            format_configuration_value(configuration.tracing.writer_options[:flush_interval])
+          options['logger.instance'] = configuration.logger.instance.class.to_s
+          options['appsec.enabled'] = configuration.dig('appsec', 'enabled') if configuration.respond_to?('appsec')
+          options['ci.enabled'] = configuration.dig('ci', 'enabled') if configuration.respond_to?('ci')
+          options['tracing.opentelemetry.enabled'] = !defined?(Datadog::OpenTelemetry::LOADED).nil?
+          options.compact!
+          options
+        end
+
+        def format_configuration_value(value)
+          # TODO: Add float if telemetry starts accepting it
+          case value
+          when Integer, String, true, false, nil
+            value
+          else
+            value.to_s
+          end
+        end
+
+        def env
+          Datadog.configuration.env
+        end
+
+        def service_name
+          Datadog.configuration.service
+        end
+
+        def service_version
+          Datadog.configuration.version
+        end
+
+        def library_version
+          Core::Environment::Identity.gem_datadog_version
+        end
+
+        def products
+          Telemetry::V1::Product.new(profiler: profiler, appsec: appsec)
+        end
+
+        def profiler
+          { version: library_version }
+        end
+
+        def appsec
+          { version: library_version }
+        end
+
+        def agent_transport
+          adapter = Core::Configuration::AgentSettingsResolver.call(Datadog.configuration).adapter
+          if adapter == Datadog::Core::Transport::Ext::UnixSocket::ADAPTER
+            'UDS'
+          else
+            'TCP'
+          end
+        end
+
+        def instrumented_integrations
+          Datadog.configuration.tracing.instrumented_integrations
+        end
+
+        def instrumented?(integration)
+          instrumented_integrations.include?(integration.name)
+        end
+
+        def patched?(integration)
+          !!integration.klass.patcher.patch_successful
+        end
+
+        def integration_auto_instrument?(integration)
+          integration.klass.auto_instrument?
+        end
+
+        def integration_compatible?(integration)
+          integration.klass.class.compatible?
+        end
+
+        def integration_version(integration)
+          integration.klass.class.version ? integration.klass.class.version.to_s : nil
+        end
+
+        def patch_error(integration)
+          patch_error_result = integration.klass.patcher.patch_error_result
+          if patch_error_result.nil? # if no error occurred during patching, but integration is still not instrumented
+            desc = "Available?: #{integration.klass.class.available?}"
+            desc += ", Loaded? #{integration.klass.class.loaded?}"
+            desc += ", Compatible? #{integration.klass.class.compatible?}"
+            desc += ", Patchable? #{integration.klass.class.patchable?}"
+            desc
+          else
+            patch_error_result.compact.to_s
+          end
+        end
+      end
+    end
+  end
+end