datadog 2.4.0 → 2.44.0

data/lib/datadog/core/workers/polling.rb

@@ -1,12 +1,14 @@
 # frozen_string_literal: true
 
-require_relative 'async'
-require_relative 'interval_loop'
+require_relative "async"
+require_relative "interval_loop"
 
 module Datadog
   module Core
     module Workers
       # Adds polling (async looping) behavior to workers
+      #
+      # @api private
       module Polling
         DEFAULT_SHUTDOWN_TIMEOUT = 1
 

data/lib/datadog/core/workers/queue.rb

@@ -5,6 +5,17 @@ module Datadog
     module Workers
       # Adds queue behavior to workers, with a buffer
       # to which items can be queued then dequeued.
+      #
+      # This module is included in some but not all workers.
+      # Notably, Data Streams Processor uses a queue but implements it
+      # inline rather than using this module.
+      #
+      # The workers that do include Queue also include Polling, which
+      # in turn includes Async::Thread and IntervalLoop. This means
+      # we have e.g. +in_iteration?+ always available in any worker
+      # that includes Queue.
+      #
+      # @api private
       module Queue
         def self.included(base)
           base.prepend(PrependedMethods)
@@ -13,11 +24,16 @@ module Datadog
         # Methods that must be prepended
         module PrependedMethods
           def perform(*args)
-            super(*dequeue) if work_pending?
+            if work_pending?
+              work = dequeue
+              super(*work)
+            end
           end
         end
 
         def buffer
+          # Why is this an unsynchronized Array and not a Core::Buffer
+          # instance?
           @buffer ||= []
         end
 
@@ -34,10 +50,89 @@ module Datadog
           !buffer.empty?
         end
 
+        # Wait for the worker to finish handling all work that has already
+        # been submitted to it.
+        #
+        # If the worker is not enabled, returns nil.
+        # If the worker is enabled, returns whether, at the point of return,
+        # there was no pending or in progress work.
+        #
+        # Flushing can time out because there is a constant stream of work
+        # submitted at the same or higher rate than it is processed.
+        # Flushing can also fail if the worker thread is not running -
+        # this method will not flush from the calling thread.
+        def flush(timeout: nil)
+          # Default timeout is 5 seconds.
+          # Specific workers can override it to be more or less
+          timeout ||= 5
+
+          # Nothing needs to be done if the worker is not enabled.
+          return nil unless enabled?
+
+          unless running?
+            unless buffer.empty?
+              # If we are asked to flush but the worker is not running,
+              # do not flush from the caller thread. If the buffer is not
+              # empty, it will not be flushed. Log a warning to this effect.
+              #
+              # We are not guaranteed to have a logger as an instance method,
+              # reference the global for now - all other worker methods
+              # also reference the logger globally.
+              # TODO inject it into worker instances.
+              Datadog.logger.debug { "Asked to flush #{self} when the worker is not running" }
+              return false
+            end
+          end
+
+          started = Utils::Time.get_time
+          loop do
+            # The AppStarted event is triggered by the worker itself,
+            # from the worker thread. As such the main thread has no way
+            # to delay itself until that event is queued and we need some
+            # way to wait until that event is sent out to assert on it in
+            # the test suite. Check the run once flag which *should*
+            # indicate the event has been queued (at which point our queue
+            # depth check should wait until it's sent).
+            # This is still a hack because the flag can be overridden
+            # either way with or without the event being sent out.
+            # Note that if the AppStarted sending fails, this check
+            # will return false and flushing will be blocked until the
+            # 15 second timeout.
+            # Note that the first wait interval between telemetry event
+            # sending is 10 seconds, the timeout needs to be strictly
+            # greater than that.
+            return true if idle?
+
+            return false if Utils::Time.get_time - started > timeout
+
+            sleep 0.5
+          end
+        end
+
         protected
 
         attr_writer \
           :buffer
+
+        # Returns whether this worker has no pending work and is not actively
+        # working.
+        #
+        # The reason why "actively working" is considered is that we use
+        # flushing to ensure all work is completed before asserting on the
+        # outcome in the tests - if work is happening in a background thread,
+        # it's too early to assert on its results.
+        def idle?
+          # We have a +work_pending?+ method in this class that semantically
+          # would be appropriate here instead of calling +buffer.empty?+.
+          # Unfortunately IntervalLoop replaces our implementation of
+          # +work_pending?+ with one that doesn't make sense at least for the
+          # Queue. And we can't change the order of module includes because
+          # they all override +perform+ and the correct behavior depends on
+          # placing IntervalLoop after Queue.
+          #
+          # Until this mess is untangled, call +buffer.empty?+ here.
+          buffer.empty? && !in_iteration?
+        end
       end
     end
   end

data/lib/datadog/core/workers/runtime_metrics.rb

@@ -1,10 +1,10 @@
 # frozen_string_literal: true
 
-require_relative '../runtime/metrics'
+require_relative "../runtime/metrics"
 
-require_relative '../worker'
-require_relative 'async'
-require_relative 'polling'
+require_relative "../worker"
+require_relative "async"
+require_relative "polling"
 
 module Datadog
   module Core
@@ -20,8 +20,16 @@ module Datadog
         attr_reader \
           :metrics
 
-        def initialize(options = {})
-          @metrics = options.fetch(:metrics) { Core::Runtime::Metrics.new }
+        def initialize(telemetry:, **options)
+          @metrics = options.fetch(:metrics) do
+            Core::Runtime::Metrics.new(
+              logger: options[:logger],
+              telemetry: telemetry,
+              experimental_propagate_process_tags_enabled: options.fetch(:propagate_process_tags_enabled) do
+                options.fetch(:experimental_propagate_process_tags_enabled)
+              end
+            )
+          end
 
           # Workers::Async::Thread settings
           self.fork_policy = options.fetch(:fork_policy, Workers::Async::Thread::FORK_POLICY_STOP)

data/lib/datadog/core.rb

@@ -1,7 +1,11 @@
 # frozen_string_literal: true
 
-require_relative 'core/deprecations'
-require_relative 'core/extensions'
+# Keep this at the top, this is needed at require-time by some files
+require_relative "ruby_version"
+
+require_relative "core/deprecations"
+require_relative "core/configuration/config_helper"
+require_relative "core/extensions"
 
 # We must load core extensions to make certain global APIs
 # accessible: both for Datadog features and the core itself.
@@ -11,6 +15,14 @@ module Datadog
   # for higher-level features.
   module Core
     extend Core::Deprecations
+
+    LIBDATADOG_API_FAILURE =
+      begin
+        require "libdatadog_api.#{RUBY_VERSION[/\d+.\d+/]}_#{RUBY_PLATFORM}"
+        nil
+      rescue LoadError => e
+        "#{e.class}: #{e.message}"
+      end
   end
 
   extend Core::Extensions
@@ -19,9 +31,14 @@ module Datadog
   # Ensures the Datadog components have a chance to gracefully
   # shut down and cleanup before terminating the process.
   at_exit do
-    if Interrupt === $! # rubocop:disable Style/SpecialGlobalVars is process terminating due to a ctrl+c or similar?
+    exception = $! # rubocop:disable Style/SpecialGlobalVars
+
+    if Interrupt === exception # is process terminating due to a ctrl+c or similar?
       Datadog.send(:handle_interrupt_shutdown!)
     else
+      # Report unhandled exception to crash tracker before shutdown
+      Datadog::Core::Crashtracking::Component.report_unhandled_exception(exception)
+
       Datadog.shutdown!
     end
   end

data/lib/datadog/data_streams/configuration.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require_relative "../core/environment/variable_helpers"
+require_relative "ext"
+
+module Datadog
+  module DataStreams
+    # Configuration for Data Streams Monitoring
+    module Configuration
+      # Configuration settings for Data Streams Monitoring.
+      module Settings
+        def self.extended(base)
+          base = base.singleton_class unless base.is_a?(Class)
+          add_settings!(base)
+        end
+
+        def self.add_settings!(base)
+          base.class_eval do
+            # Data Streams Monitoring configuration
+            # @public_api
+            settings :data_streams do
+              # Whether Data Streams Monitoring is enabled. When enabled, the library will
+              # collect and report data lineage information for messaging systems.
+              #
+              # @default `DD_DATA_STREAMS_ENABLED` environment variable, otherwise `false`.
+              # @return [Boolean]
+              option :enabled do |o|
+                o.type :bool
+                o.env Ext::ENV_ENABLED
+                o.default false
+              end
+
+              # The interval (in seconds) at which Data Streams Monitoring stats are flushed.
+              #
+              # @default 10.0
+              # @env '_DD_TRACE_STATS_WRITER_INTERVAL'
+              # @return [Float]
+              # @!visibility private
+              option :interval do |o|
+                o.type :float
+                o.env "_DD_TRACE_STATS_WRITER_INTERVAL"
+                o.default 10.0
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/data_streams/ext.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DataStreams
+    # Constants for Data Streams Monitoring configuration
+    # @public_api
+    module Ext
+      ENV_ENABLED = "DD_DATA_STREAMS_ENABLED"
+    end
+  end
+end

data/lib/datadog/data_streams/extensions.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require_relative "configuration"
+require_relative "../core/configuration"
+
+module Datadog
+  module DataStreams
+    # Extends Datadog with Data Streams Monitoring features
+    module Extensions
+      # Inject Data Streams settings into global configuration.
+      def self.activate!
+        Core::Configuration::Settings.extend(Configuration::Settings)
+      end
+    end
+  end
+end

data/lib/datadog/data_streams/pathway_context.rb

@@ -0,0 +1,167 @@
+# frozen_string_literal: true
+
+require "stringio"
+require_relative "../core/utils/base64_codec"
+
+module Datadog
+  module DataStreams
+    # Represents a pathway context for data streams monitoring
+    class PathwayContext
+      # The current pathway hash value (result of FNV-1a hash function)
+      attr_accessor :hash
+      # When the pathway started
+      attr_accessor :pathway_start
+      # When the current edge started
+      attr_accessor :current_edge_start
+      # The hash value of the parent checkpoint
+      attr_accessor :parent_hash
+      # The direction tag of the previous checkpoint (e.g., 'direction:in', 'direction:out'), or nil if none
+      attr_accessor :previous_direction
+      # Hash value of the closest checkpoint in opposite direction (used for loop detection)
+      attr_accessor :closest_opposite_direction_hash
+      # Edge start time of the closest opposite direction checkpoint
+      attr_accessor :closest_opposite_direction_edge_start
+
+      def initialize(hash_value:, pathway_start:, current_edge_start:)
+        @hash = hash_value
+        @pathway_start = pathway_start
+        @current_edge_start = current_edge_start
+        @parent_hash = nil
+
+        @previous_direction = nil
+        @closest_opposite_direction_hash = 0
+        @closest_opposite_direction_edge_start = current_edge_start
+      end
+
+      def encode_b64
+        Core::Utils::Base64Codec.strict_encode64(encode)
+      end
+
+      # Decode pathway context from base64 encoded string
+      def self.decode_b64(encoded_ctx)
+        return nil unless encoded_ctx && !encoded_ctx.empty?
+
+        begin
+          binary_data = Core::Utils::Base64Codec.strict_decode64(encoded_ctx)
+          decode(binary_data)
+        rescue ArgumentError => e
+          # Invalid base64 encoding - may indicate version mismatch or corruption
+          Datadog.logger.debug { "Failed to decode DSM pathway context: #{e.class}: #{e.message}" }
+          nil
+        end
+      end
+
+      private
+
+      def encode
+        # Format:
+        # - 8 bytes: hash value (little-endian)
+        # - VarInt: pathway start time (milliseconds)
+        # - VarInt: current edge start time (milliseconds)
+        [@hash].pack("Q") <<
+          encode_var_int_64(time_to_ms(@pathway_start)) <<
+          encode_var_int_64(time_to_ms(@current_edge_start))
+      end
+
+      # Decode pathway context from binary data
+      def self.decode(binary_data)
+        return nil unless binary_data && binary_data.bytesize >= 8
+
+        reader = StringIO.new(binary_data)
+
+        # Extract 8-byte hash (little-endian)
+        hash_bytes = reader.read(8)
+        return nil unless hash_bytes
+
+        hash_value = hash_bytes.unpack1("Q") # : Integer
+
+        # Extract pathway start time (VarInt milliseconds)
+        pathway_start_ms = decode_varint(reader)
+        return nil unless pathway_start_ms
+
+        # Extract current edge start time (VarInt milliseconds)
+        current_edge_start_ms = decode_varint(reader)
+        return nil unless current_edge_start_ms
+
+        # Convert milliseconds to Time objects
+        pathway_start = ms_to_time(pathway_start_ms)
+        current_edge_start = ms_to_time(current_edge_start_ms)
+
+        new(
+          hash_value: hash_value,
+          pathway_start: pathway_start,
+          current_edge_start: current_edge_start
+        )
+      rescue EOFError
+        # Not enough data in binary stream
+        nil
+      end
+      private_class_method :decode
+
+      # Encode a signed 64-bit integer using ZigZag + LEB128 variable-length encoding.
+      #
+      # dd-trace-go's DSM pathway encoder (internal/datastreams/propagator.go, via
+      # sketches-go's ddsketch/encoding package) ZigZag-maps the signed value before
+      # applying plain unsigned LEB128 (DWARF5 section 7.6:
+      # https://dwarfstd.org/doc/DWARF5.pdf#page=301). This implementation previously
+      # skipped the ZigZag step, which silently decoded to the wrong timestamp on any
+      # dd-trace-go consumer -- see https://github.com/DataDog/dd-trace-go/issues/5163.
+      #
+      # ZigZag maps signed values to unsigned ones so small-magnitude negative numbers
+      # still encode to few bytes: 0, -1, 1, -2, 2, ... -> 0, 1, 2, 3, 4, ...
+      #
+      # @param value [Integer] Signed integer value to encode
+      # @return [String] Binary string of encoded bytes
+      def encode_var_int_64(value)
+        zigzag = (value << 1) ^ (value >> 63)
+        bytes = []
+        while zigzag >= 0x80
+          bytes << ((zigzag & 0x7F) | 0x80)
+          zigzag >>= 7
+        end
+        bytes << zigzag
+        bytes.pack("C*")
+      end
+
+      # Decode a signed 64-bit integer encoded with ZigZag + LEB128 from an IO stream.
+      #
+      # See #encode_var_int_64 for why the ZigZag un-map is required to match
+      # dd-trace-go's wire format.
+      #
+      # @param io [StringIO] IO stream to read from
+      # @return [Integer, nil] Decoded signed integer, or nil if malformed
+      def self.decode_varint(io)
+        value = 0
+        shift = 0
+
+        loop do
+          byte = io.readbyte
+
+          # Add this byte's 7 data bits to our value
+          value |= (byte & 0x7F) << shift
+
+          # If high bit is clear, we're done
+          return (value >> 1) ^ -(value & 1) unless (byte & 0x80).nonzero?
+
+          shift += 7
+
+          # Safety: prevent infinite decoding
+          return nil if shift >= 64
+        end
+      rescue EOFError
+        # Stream ended unexpectedly - malformed data
+        nil
+      end
+      private_class_method :decode_varint
+
+      def self.ms_to_time(milliseconds)
+        ::Time.at(milliseconds / 1000.0)
+      end
+      private_class_method :ms_to_time
+
+      def time_to_ms(time)
+        (time.to_f * 1000).to_i
+      end
+    end
+  end
+end