datadog 2.0.0 → 2.2.0

data/lib/datadog/core/telemetry/metric.rb

@@ -0,0 +1,167 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Telemetry
+      # Telemetry metrics data model (internal Datadog metrics for client libraries)
+      module Metric
+        # Base class for all metric types
+        class Base
+          attr_reader :name, :tags, :values, :common
+
+          # @param name [String] metric name
+          # @param tags [Array<String>|Hash{String=>String}] metric tags as hash or array of "tag:val" strings
+          # @param common [Boolean] true if the metric is common for all languages, false for Ruby-specific metric
+          def initialize(name, tags: {}, common: true)
+            @name = name
+            @values = []
+            @tags = tags_to_array(tags)
+            @common = common
+          end
+
+          def id
+            @id ||= "#{type}::#{name}::#{tags.join(',')}"
+          end
+
+          def track(value)
+            raise NotImplementedError, 'method must be implemented in subclasses'
+          end
+
+          def type
+            raise NotImplementedError, 'method must be implemented in subclasses'
+          end
+
+          def to_h
+            {
+              metric: name,
+              points: values,
+              type: type,
+              tags: tags,
+              common: common
+            }
+          end
+
+          private
+
+          def tags_to_array(tags)
+            return tags if tags.is_a?(Array)
+
+            tags.map { |k, v| "#{k}:#{v}" }
+          end
+        end
+
+        # Base class for metrics that require aggregation interval
+        class IntervalMetric < Base
+          attr_reader :interval
+
+          # @param name [String] metric name
+          # @param tags [Array<String>|Hash{String=>String}] metric tags as hash of array of "tag:val" strings
+          # @param common [Boolean] true if the metric is common for all languages, false for Ruby-specific metric
+          # @param interval [Integer] metrics aggregation interval in seconds
+          def initialize(name, interval:, tags: {}, common: true)
+            raise ArgumentError, 'interval must be a positive number' if interval.nil? || interval <= 0
+
+            super(name, tags: tags, common: common)
+
+            @interval = interval
+          end
+
+          def to_h
+            res = super
+            res[:interval] = interval
+            res
+          end
+        end
+
+        # Count metric adds up all the submitted values in a time interval. This would be suitable for a
+        # metric tracking the number of website hits, for instance.
+        class Count < Base
+          TYPE = 'count'
+
+          def type
+            TYPE
+          end
+
+          def track(value)
+            value = value.to_i
+
+            if values.empty?
+              values << [Time.now.to_i, value]
+            else
+              values[0][0] = Time.now.to_i
+              values[0][1] += value
+            end
+            nil
+          end
+        end
+
+        # A gauge type takes the last value reported during the interval. This type would make sense for tracking RAM or
+        # CPU usage, where taking the last value provides a representative picture of the host’s behavior during the time
+        # interval.
+        class Gauge < IntervalMetric
+          TYPE = 'gauge'
+
+          def type
+            TYPE
+          end
+
+          def track(value)
+            if values.empty?
+              values << [Time.now.to_i, value]
+            else
+              values[0][0] = Time.now.to_i
+              values[0][1] = value
+            end
+            nil
+          end
+        end
+
+        # The rate type takes the count and divides it by the length of the time interval. This is useful if you’re
+        # interested in the number of hits per second.
+        class Rate < IntervalMetric
+          TYPE = 'rate'
+
+          def initialize(name, interval:, tags: {}, common: true)
+            super
+
+            @value = 0.0
+          end
+
+          def type
+            TYPE
+          end
+
+          def track(value = 1.0)
+            @value += value
+            @values = [[Time.now.to_i, @value / interval]]
+            nil
+          end
+        end
+
+        # Distribution metric represents the global statistical distribution of a set of values.
+        class Distribution < Base
+          TYPE = 'distributions'
+
+          def type
+            TYPE
+          end
+
+          def track(value)
+            values << value
+            nil
+          end
+
+          # distribution metric data does not have type field
+          def to_h
+            {
+              metric: name,
+              points: values,
+              tags: tags,
+              common: common
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/metrics_collection.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require_relative 'event'
+require_relative 'metric'
+
+module Datadog
+  module Core
+    module Telemetry
+      # MetricsCollection is a thread-safe collection of metrics per namespace
+      class MetricsCollection
+        attr_reader :namespace, :interval
+
+        def initialize(namespace, aggregation_interval:)
+          @namespace = namespace
+          @interval = aggregation_interval
+
+          @mutex = Mutex.new
+
+          @metrics = {}
+          @distributions = {}
+        end
+
+        def inc(metric_name, value, tags: {}, common: true)
+          metric = Metric::Count.new(metric_name, tags: tags, common: common)
+          fetch_or_add_metric(metric, value)
+        end
+
+        def dec(metric_name, value, tags: {}, common: true)
+          metric = Metric::Count.new(metric_name, tags: tags, common: common)
+          fetch_or_add_metric(metric, -value)
+        end
+
+        def gauge(metric_name, value, tags: {}, common: true)
+          metric = Metric::Gauge.new(metric_name, tags: tags, common: common, interval: @interval)
+          fetch_or_add_metric(metric, value)
+        end
+
+        def rate(metric_name, value, tags: {}, common: true)
+          metric = Metric::Rate.new(metric_name, tags: tags, common: common, interval: @interval)
+          fetch_or_add_metric(metric, value)
+        end
+
+        def distribution(metric_name, value, tags: {}, common: true)
+          metric = Metric::Distribution.new(metric_name, tags: tags, common: common)
+          fetch_or_add_distribution(metric, value)
+        end
+
+        def flush!
+          @mutex.synchronize do
+            events = []
+            events << Event::GenerateMetrics.new(@namespace, @metrics.values) if @metrics.any?
+            events << Event::Distributions.new(@namespace, @distributions.values) if @distributions.any?
+
+            @metrics = {}
+            @distributions = {}
+
+            events
+          end
+        end
+
+        private
+
+        def fetch_or_add_metric(metric, value)
+          @mutex.synchronize do
+            m = (@metrics[metric.id] ||= metric)
+            m.track(value)
+          end
+          nil
+        end
+
+        def fetch_or_add_distribution(metric, value)
+          @mutex.synchronize do
+            m = (@distributions[metric.id] ||= metric)
+            m.track(value)
+          end
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/metrics_manager.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require_relative 'metrics_collection'
+
+module Datadog
+  module Core
+    module Telemetry
+      # MetricsManager aggregates and flushes metrics and distributions
+      class MetricsManager
+        attr_reader :enabled
+
+        def initialize(aggregation_interval:, enabled:)
+          @interval = aggregation_interval
+          @enabled = enabled
+          @mutex = Mutex.new
+
+          @collections = {}
+        end
+
+        def inc(namespace, metric_name, value, tags: {}, common: true)
+          return unless @enabled
+
+          # collection is thread-safe internally
+          collection = fetch_or_create_collection(namespace)
+          collection.inc(metric_name, value, tags: tags, common: common)
+        end
+
+        def dec(namespace, metric_name, value, tags: {}, common: true)
+          return unless @enabled
+
+          # collection is thread-safe internally
+          collection = fetch_or_create_collection(namespace)
+          collection.dec(metric_name, value, tags: tags, common: common)
+        end
+
+        def gauge(namespace, metric_name, value, tags: {}, common: true)
+          return unless @enabled
+
+          # collection is thread-safe internally
+          collection = fetch_or_create_collection(namespace)
+          collection.gauge(metric_name, value, tags: tags, common: common)
+        end
+
+        def rate(namespace, metric_name, value, tags: {}, common: true)
+          return unless @enabled
+
+          # collection is thread-safe internally
+          collection = fetch_or_create_collection(namespace)
+          collection.rate(metric_name, value, tags: tags, common: common)
+        end
+
+        def distribution(namespace, metric_name, value, tags: {}, common: true)
+          return unless @enabled
+
+          # collection is thread-safe internally
+          collection = fetch_or_create_collection(namespace)
+          collection.distribution(metric_name, value, tags: tags, common: common)
+        end
+
+        def flush!
+          return [] unless @enabled
+
+          collections = @mutex.synchronize { @collections.values }
+          collections.reduce([]) { |events, collection| events + collection.flush! }
+        end
+
+        def disable!
+          @enabled = false
+        end
+
+        private
+
+        def fetch_or_create_collection(namespace)
+          @mutex.synchronize do
+            @collections[namespace] ||= MetricsCollection.new(namespace, aggregation_interval: @interval)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/request.rb

@@ -17,7 +17,7 @@ module Datadog
               application: application,
               debug: false,
               host: host,
-              payload: event.payload(seq_id),
+              payload: event.payload,
               request_type: event.type,
               runtime_id: Core::Environment::Identity.id,
               seq_id: seq_id,

data/lib/datadog/core/telemetry/worker.rb

@@ -0,0 +1,173 @@
+# frozen_string_literal: true
+
+require_relative 'event'
+
+require_relative '../utils/only_once_successful'
+require_relative '../workers/polling'
+require_relative '../workers/queue'
+
+module Datadog
+  module Core
+    module Telemetry
+      # Accumulates events and sends them to the API at a regular interval, including heartbeat event.
+      class Worker
+        include Core::Workers::Queue
+        include Core::Workers::Polling
+
+        DEFAULT_BUFFER_MAX_SIZE = 1000
+        APP_STARTED_EVENT_RETRIES = 10
+
+        TELEMETRY_STARTED_ONCE = Utils::OnlyOnceSuccessful.new(APP_STARTED_EVENT_RETRIES)
+
+        def initialize(
+          heartbeat_interval_seconds:,
+          metrics_aggregation_interval_seconds:,
+          emitter:,
+          metrics_manager:,
+          dependency_collection:,
+          enabled: true,
+          shutdown_timeout: Workers::Polling::DEFAULT_SHUTDOWN_TIMEOUT,
+          buffer_size: DEFAULT_BUFFER_MAX_SIZE
+        )
+          @emitter = emitter
+          @metrics_manager = metrics_manager
+          @dependency_collection = dependency_collection
+
+          @ticks_per_heartbeat = (heartbeat_interval_seconds / metrics_aggregation_interval_seconds).to_i
+          @current_ticks = 0
+
+          # Workers::Polling settings
+          self.enabled = enabled
+          # Workers::IntervalLoop settings
+          self.loop_base_interval = metrics_aggregation_interval_seconds
+          self.fork_policy = Core::Workers::Async::Thread::FORK_POLICY_STOP
+
+          @shutdown_timeout = shutdown_timeout
+          @buffer_size = buffer_size
+
+          self.buffer = buffer_klass.new(@buffer_size)
+        end
+
+        def start
+          return if !enabled? || forked?
+
+          # starts async worker
+          perform
+        end
+
+        def stop(force_stop = false, timeout = @shutdown_timeout)
+          buffer.close if running?
+
+          super
+        end
+
+        def enqueue(event)
+          return if !enabled? || forked?
+
+          buffer.push(event)
+        end
+
+        def sent_started_event?
+          TELEMETRY_STARTED_ONCE.success?
+        end
+
+        def failed_to_start?
+          TELEMETRY_STARTED_ONCE.failed?
+        end
+
+        private
+
+        def perform(*events)
+          return if !enabled? || forked?
+
+          started! unless sent_started_event?
+
+          metric_events = @metrics_manager.flush!
+          events = [] if events.nil?
+          flush_events(events + metric_events)
+
+          @current_ticks += 1
+          return if @current_ticks < @ticks_per_heartbeat
+
+          @current_ticks = 0
+          heartbeat!
+        end
+
+        def flush_events(events)
+          return if events.empty?
+          return if !enabled? || !sent_started_event?
+
+          Datadog.logger.debug { "Sending #{events&.count} telemetry events" }
+          send_event(Event::MessageBatch.new(events))
+        end
+
+        def heartbeat!
+          return if !enabled? || !sent_started_event?
+
+          send_event(Event::AppHeartbeat.new)
+        end
+
+        def started!
+          return unless enabled?
+
+          if failed_to_start?
+            Datadog.logger.debug('Telemetry app-started event exhausted retries, disabling telemetry worker')
+            disable!
+            return
+          end
+
+          TELEMETRY_STARTED_ONCE.run do
+            res = send_event(Event::AppStarted.new)
+
+            if res.ok?
+              Datadog.logger.debug('Telemetry app-started event is successfully sent')
+
+              send_event(Event::AppDependenciesLoaded.new) if @dependency_collection
+
+              true
+            else
+              Datadog.logger.debug('Error sending telemetry app-started event, retry after heartbeat interval...')
+              false
+            end
+          end
+        end
+
+        def send_event(event)
+          res = @emitter.request(event)
+
+          disable_on_not_found!(res)
+
+          res
+        end
+
+        def dequeue
+          buffer.pop
+        end
+
+        def work_pending?
+          run_loop? || !buffer.empty?
+        end
+
+        def buffer_klass
+          if Core::Environment::Ext::RUBY_ENGINE == 'ruby'
+            Core::Buffer::CRuby
+          else
+            Core::Buffer::ThreadSafe
+          end
+        end
+
+        def disable!
+          self.enabled = false
+          @metrics_manager.disable!
+        end
+
+        def disable_on_not_found!(response)
+          return unless response.not_found?
+
+          Datadog.logger.debug('Agent does not support telemetry; disabling future telemetry events.')
+          disable!
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/utils/only_once_successful.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require_relative 'only_once'
+
+module Datadog
+  module Core
+    module Utils
+      # Helper class to execute something with only one success.
+      #
+      # This is useful for cases where we want to ensure that a block of code is only executed once, and only if it
+      # succeeds. One such example is sending app-started telemetry event.
+      #
+      # Successful execution is determined by the return value of the block: any truthy value is considered success.
+      #
+      # Thread-safe when used correctly (e.g. be careful of races when lazily initializing instances of this class).
+      #
+      # Note: In its current state, this class is not Ractor-safe.
+      # In https://github.com/DataDog/dd-trace-rb/pull/1398#issuecomment-797378810 we have a discussion of alternatives,
+      # including an alternative implementation that is Ractor-safe once spent.
+      class OnlyOnceSuccessful < OnlyOnce
+        def initialize(limit = 0)
+          super()
+
+          @limit = limit
+          @failed = false
+          @retries = 0
+        end
+
+        def run
+          @mutex.synchronize do
+            return if @ran_once
+
+            result = yield
+            @ran_once = !!result
+
+            if !@ran_once && limited?
+              @retries += 1
+              check_limit!
+            end
+
+            result
+          end
+        end
+
+        def success?
+          @mutex.synchronize { @ran_once && !@failed }
+        end
+
+        def failed?
+          @mutex.synchronize { @ran_once && @failed }
+        end
+
+        private
+
+        def check_limit!
+          if @retries >= @limit
+            @failed = true
+            @ran_once = true
+          end
+        end
+
+        def limited?
+          !@limit.nil? && @limit.positive?
+        end
+
+        def reset_ran_once_state_for_tests
+          @mutex.synchronize do
+            @ran_once = false
+            @failed = false
+            @retries = 0
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative 'core/deprecations'
 require_relative 'core/extensions'
 
 # We must load core extensions to make certain global APIs
@@ -9,25 +10,7 @@ module Datadog
   # products. It is a dependency of each product. Contrast with Datadog::Kit
   # for higher-level features.
   module Core
-    class << self
-      # Records the occurrence of a deprecated operation in this library.
-      #
-      # Currently, these operations are logged to `Datadog.logger` at `warn` level.
-      #
-      # `disallowed_next_major` adds a message informing that the deprecated operation
-      # won't be allowed in the next major release.
-      #
-      # @yieldreturn [String] a String with the lazily evaluated deprecation message.
-      # @param [Boolean] disallowed_next_major whether this deprecation will be enforced in the next major release.
-      def log_deprecation(disallowed_next_major: true)
-        Datadog.logger.warn do
-          message = yield
-          message += ' This will be enforced in the next major release.' if disallowed_next_major
-          message
-        end
-        nil
-      end
-    end
+    extend Core::Deprecations
   end
 
   extend Core::Extensions

data/lib/datadog/opentelemetry/sdk/propagator.rb

@@ -41,8 +41,11 @@ module Datadog
           digest = @datadog_propagator.extract(carrier)
           return context unless digest
 
-          trace_id = to_otel_id(digest.trace_id)
-          span_id = to_otel_id(digest.span_id)
+          # Converts the {Numeric} Datadog id object to OpenTelemetry's byte array format.
+          # 128-bit unsigned, big-endian integer
+          trace_id = [digest.trace_id >> 64, digest.trace_id & 0xFFFFFFFFFFFFFFFF].pack('Q>Q>')
+          # 64-bit unsigned, big-endian integer
+          span_id = [digest.span_id].pack('Q>')
 
           if digest.trace_state || digest.trace_flags
             trace_flags = ::OpenTelemetry::Trace::TraceFlags.from_byte(digest.trace_flags)
@@ -78,14 +81,6 @@ module Datadog
         def fields
           []
         end
-
-        private
-
-        # Converts the {Numeric} Datadog id object to OpenTelemetry's byte array format.
-        # This method currently converts an unsigned 64-bit Integer to a binary String.
-        def to_otel_id(dd_id)
-          Array(dd_id).pack('Q')
-        end
       end
     end
   end

data/lib/datadog/opentelemetry/sdk/span_processor.rb

@@ -70,7 +70,8 @@ module Datadog
           if parent_context.trace
             Tracing.send(:tracer).send(:call_context).activate!(parent_context.ensure_trace)
           else
-            Tracing.continue_trace!(nil)
+            otel_trace_id = span.context.hex_trace_id.to_i(16)
+            Tracing.continue_trace!(Datadog::Tracing::TraceDigest.new(trace_id: otel_trace_id, span_remote: false))
           end
 
           datadog_span = start_datadog_span(span)
@@ -85,7 +86,6 @@ module Datadog
           name, kwargs = span_arguments(span, attributes)
 
           datadog_span = Tracing.trace(name, **kwargs)
-
           datadog_span.set_error([nil, span.status.description]) unless span.status.ok?
           datadog_span.set_tags(span.attributes)
 
@@ -143,6 +143,9 @@ module Datadog
 
           kwargs[:tags] = attributes.to_h
 
+          # DEV: The datadog span must have the same ID as the OpenTelemetry span
+          kwargs[:id] = span.context.hex_span_id.to_i(16)
+
           [name, kwargs]
         end