dogstatsd-ruby 4.0.0 → 5.3.3

data/lib/datadog/statsd/sender.rb

@@ -0,0 +1,177 @@
+# frozen_string_literal: true
+
+module Datadog
+  class Statsd
+    # Sender is using a companion thread to flush and pack messages
+    # in a `MessageBuffer`.
+    # The communication with this thread is done using a `Queue`.
+    # If the thread is dead, it is starting a new one to avoid having a blocked
+    # Sender with no companion thread to communicate with (most of the time, having
+    # a dead companion thread means that a fork just happened and that we are
+    # running in the child process).
+    class Sender
+      CLOSEABLE_QUEUES = Queue.instance_methods.include?(:close)
+
+      def initialize(message_buffer, telemetry: nil, queue_size: UDP_DEFAULT_BUFFER_SIZE, logger: nil, flush_interval: nil, queue_class: Queue, thread_class: Thread)
+        @message_buffer = message_buffer
+        @telemetry = telemetry
+        @queue_size = queue_size
+        @logger = logger
+        @mx = Mutex.new
+        @queue_class = queue_class
+        @thread_class = thread_class
+        if flush_interval
+          @flush_timer = Datadog::Statsd::Timer.new(flush_interval) { flush(sync: true) }
+        end
+      end
+
+      def flush(sync: false)
+        # keep a copy around in case another thread is calling #stop while this method is running
+        current_message_queue = message_queue
+
+        # don't try to flush if there is no message_queue instantiated or
+        # no companion thread running
+        if !current_message_queue
+          @logger.debug { "Statsd: can't flush: no message queue ready" } if @logger
+          return
+        end
+        if !sender_thread.alive?
+          @logger.debug { "Statsd: can't flush: no sender_thread alive" } if @logger
+          return
+        end
+
+        current_message_queue.push(:flush)
+        rendez_vous if sync
+      end
+
+      def rendez_vous
+        # could happen if #start hasn't be called
+        return unless message_queue
+
+        # Initialize and get the thread's sync queue
+        queue = (@thread_class.current[:statsd_sync_queue] ||= @queue_class.new)
+        # tell sender-thread to notify us in the current
+        # thread's queue
+        message_queue.push(queue)
+        # wait for the sender thread to send a message
+        # once the flush is done
+        queue.pop
+      end
+
+      def add(message)
+        raise ArgumentError, 'Start sender first' unless message_queue
+
+        # if the thread does not exist, we assume we are running in a forked process,
+        # empty the message queue and message buffers (these messages belong to
+        # the parent process) and spawn a new companion thread.
+        if !sender_thread.alive?
+          @mx.synchronize {
+            # a call from another thread has already re-created
+            # the companion thread before this one acquired the lock
+            break if sender_thread.alive?
+            @logger.debug { "Statsd: companion thread is dead, re-creating one" } if @logger
+
+            message_queue.close if CLOSEABLE_QUEUES
+            @message_queue = nil
+            message_buffer.reset
+            start
+            @flush_timer.start if @flush_timer && @flush_timer.stop?
+          }
+        end
+
+        if message_queue.length <= @queue_size
+          message_queue << message
+        else
+          @telemetry.dropped_queue(packets: 1, bytes: message.bytesize) if @telemetry
+        end
+      end
+
+      def start
+        raise ArgumentError, 'Sender already started' if message_queue
+
+        # initialize a new message queue for the background thread
+        @message_queue = @queue_class.new
+        # start background thread
+        @sender_thread = @thread_class.new(&method(:send_loop))
+        @sender_thread.name = "Statsd Sender" unless Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.3')
+        @flush_timer.start if @flush_timer
+      end
+
+      if CLOSEABLE_QUEUES
+        # when calling stop, make sure that no other threads is trying
+        # to close the sender nor trying to continue to `#add` more message
+        # into the sender.
+        def stop(join_worker: true)
+          message_queue = @message_queue
+          message_queue.close if message_queue
+
+          @flush_timer.stop if @flush_timer
+          sender_thread = @sender_thread
+          sender_thread.join if sender_thread && join_worker
+        end
+      else
+        # when calling stop, make sure that no other threads is trying
+        # to close the sender nor trying to continue to `#add` more message
+        # into the sender.
+        def stop(join_worker: true)
+          message_queue = @message_queue
+          message_queue << :close if message_queue
+
+          @flush_timer.stop if @flush_timer
+          sender_thread = @sender_thread
+          sender_thread.join if sender_thread && join_worker
+        end
+      end
+
+      private
+
+      attr_reader :message_buffer
+      attr_reader :message_queue
+      attr_reader :sender_thread
+
+      if CLOSEABLE_QUEUES
+        def send_loop
+          until (message = message_queue.pop).nil? && message_queue.closed?
+            # skip if message is nil, e.g. when message_queue
+            # is empty and closed
+            next unless message
+
+            case message
+            when :flush
+              message_buffer.flush
+            when @queue_class
+              message.push(:go_on)
+            else
+              message_buffer.add(message)
+            end
+          end
+
+          @message_queue = nil
+          @sender_thread = nil
+        end
+      else
+        def send_loop
+          loop do
+            message = message_queue.pop
+
+            next unless message
+
+            case message
+            when :close
+              break
+            when :flush
+              message_buffer.flush
+            when @queue_class
+              message.push(:go_on)
+            else
+              message_buffer.add(message)
+            end
+          end
+
+          @message_queue = nil
+          @sender_thread = nil
+        end
+      end
+    end
+  end
+end

data/lib/datadog/statsd/serialization/event_serializer.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Datadog
+  class Statsd
+    module Serialization
+      class EventSerializer
+        EVENT_BASIC_OPTIONS = {
+          date_happened:    'd:',
+          hostname:         'h:',
+          aggregation_key:  'k:',
+          priority:         'p:',
+          source_type_name: 's:',
+          alert_type:       't:',
+        }.freeze
+
+        def initialize(global_tags: [])
+          @tag_serializer = TagSerializer.new(global_tags)
+        end
+
+        def format(title, text, options = EMPTY_OPTIONS)
+          title = escape(title)
+          text = escape(text)
+
+          String.new.tap do |event|
+            event << '_e{'
+            event << title.bytesize.to_s
+            event << ','
+            event << text.bytesize.to_s
+            event << '}:'
+            event << title
+            event << '|'
+            event << text
+
+            # we are serializing the generic service check options
+            # before serializing specialized options that need edge-cases
+            EVENT_BASIC_OPTIONS.each do |option_key, shortcut|
+              if value = options[option_key]
+                event << '|'
+                event << shortcut
+                event << value.to_s.delete('|')
+              end
+            end
+
+            # also returns the global tags from serializer
+            if tags = tag_serializer.format(options[:tags])
+              event << '|#'
+              event << tags
+            end
+
+            if event.bytesize > MAX_EVENT_SIZE
+              if options[:truncate_if_too_long]
+                event.slice!(MAX_EVENT_SIZE..event.length)
+              else
+                raise "Event #{title} payload is too big (more that 8KB), event discarded"
+              end
+            end
+          end
+        end
+
+        protected
+        attr_reader :tag_serializer
+
+        def escape(text)
+          text.delete('|').tap do |t|
+            t.gsub!("\n", '\n')
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/statsd/serialization/serializer.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# require 'forwardable'
+
+module Datadog
+  class Statsd
+    module Serialization
+      class Serializer
+        def initialize(prefix: nil, global_tags: [])
+          @stat_serializer = StatSerializer.new(prefix, global_tags: global_tags)
+          @service_check_serializer = ServiceCheckSerializer.new(global_tags: global_tags)
+          @event_serializer = EventSerializer.new(global_tags: global_tags)
+        end
+
+        # using *args would make new allocations
+        def to_stat(name, delta, type, tags: [], sample_rate: 1)
+          stat_serializer.format(name, delta, type, tags: tags, sample_rate: sample_rate)
+        end
+
+        # using *args would make new allocations
+        def to_service_check(name, status, options = EMPTY_OPTIONS)
+          service_check_serializer.format(name, status, options)
+        end
+
+        # using *args would make new allocations
+        def to_event(title, text, options = EMPTY_OPTIONS)
+          event_serializer.format(title, text, options)
+        end
+
+        def global_tags
+          stat_serializer.global_tags
+        end
+
+        protected
+        attr_reader :stat_serializer
+        attr_reader :service_check_serializer
+        attr_reader :event_serializer
+      end
+    end
+  end
+end

data/lib/datadog/statsd/serialization/service_check_serializer.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Datadog
+  class Statsd
+    module Serialization
+      class ServiceCheckSerializer
+        SERVICE_CHECK_BASIC_OPTIONS = {
+          timestamp: 'd:',
+          hostname:  'h:',
+        }.freeze
+
+        def initialize(global_tags: [])
+          @tag_serializer = TagSerializer.new(global_tags)
+        end
+
+        def format(name, status, options = EMPTY_OPTIONS)
+          String.new.tap do |service_check|
+            # line basics
+            service_check << "_sc"
+            service_check << "|"
+            service_check << name.to_s
+            service_check << "|"
+            service_check << status.to_s
+
+            # we are serializing the generic service check options
+            # before serializing specialized options that need edge-cases
+            SERVICE_CHECK_BASIC_OPTIONS.each do |option_key, shortcut|
+              if value = options[option_key]
+                service_check << '|'
+                service_check << shortcut
+                service_check << value.to_s.delete('|')
+              end
+            end
+
+            if message = options[:message]
+              service_check << '|m:'
+              service_check << escape_message(message)
+            end
+
+            # also returns the global tags from serializer
+            if tags = tag_serializer.format(options[:tags])
+              service_check << '|#'
+              service_check << tags
+            end
+          end
+        end
+
+        protected
+        attr_reader :tag_serializer
+
+        def escape_message(message)
+          message.delete('|').tap do |m|
+            m.gsub!("\n", '\n')
+            m.gsub!('m:', 'm\:')
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/statsd/serialization/stat_serializer.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Datadog
+  class Statsd
+    module Serialization
+      class StatSerializer
+        def initialize(prefix, global_tags: [])
+          @prefix = prefix
+          @prefix_str = prefix.to_s
+          @tag_serializer = TagSerializer.new(global_tags)
+        end
+
+        def format(name, delta, type, tags: [], sample_rate: 1)
+          name = formated_name(name)
+
+          if sample_rate != 1
+            if tags_list = tag_serializer.format(tags)
+              "#{@prefix_str}#{name}:#{delta}|#{type}|@#{sample_rate}|##{tags_list}"
+            else
+              "#{@prefix_str}#{name}:#{delta}|#{type}|@#{sample_rate}"
+            end
+          else
+            if tags_list = tag_serializer.format(tags)
+              "#{@prefix_str}#{name}:#{delta}|#{type}|##{tags_list}"
+            else
+              "#{@prefix_str}#{name}:#{delta}|#{type}"
+            end
+          end
+        end
+
+        def global_tags
+          tag_serializer.global_tags
+        end
+
+        private
+
+        attr_reader :prefix
+        attr_reader :tag_serializer
+
+        def formated_name(name)
+          if name.is_a?(String)
+            # DEV: gsub is faster than dup.gsub!
+            formated = name.gsub('::', '.')
+          else
+            formated = name.to_s
+            formated.gsub!('::', '.')
+          end
+
+          formated.tr!(':|@', '_')
+          formated
+        end
+      end
+    end
+  end
+end

data/lib/datadog/statsd/serialization/tag_serializer.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module Datadog
+  class Statsd
+    module Serialization
+      class TagSerializer
+        def initialize(global_tags = [], env = ENV)
+          # Convert to hash
+          global_tags = to_tags_hash(global_tags)
+
+          # Merge with default tags
+          global_tags = default_tags(env).merge(global_tags)
+
+          # Convert to tag list and set
+          @global_tags = to_tags_list(global_tags)
+          if @global_tags.any?
+            @global_tags_formatted = @global_tags.join(',')
+          else
+            @global_tags_formatted = nil
+          end
+        end
+
+        def format(message_tags)
+          if !message_tags || message_tags.empty?
+            return @global_tags_formatted
+          end
+
+          tags = if @global_tags_formatted
+                   [@global_tags_formatted, to_tags_list(message_tags)]
+                 else
+                   to_tags_list(message_tags)
+                 end
+
+          tags.join(',')
+        end
+
+        attr_reader :global_tags
+
+        private
+
+        def to_tags_hash(tags)
+          case tags
+          when Hash
+            tags.dup
+          when Array
+            Hash[
+              tags.map do |string|
+                tokens = string.split(':')
+                tokens << nil if tokens.length == 1
+                tokens.length == 2 ? tokens : nil
+              end.compact
+            ]
+          else
+            {}
+          end
+        end
+
+        def to_tags_list(tags)
+          case tags
+          when Hash
+            tags.map do |name, value|
+              if value
+                escape_tag_content("#{name}:#{value}")
+              else
+                escape_tag_content(name)
+              end
+            end
+          when Array
+            tags.map { |tag| escape_tag_content(tag) }
+          else
+            []
+          end
+        end
+
+        def escape_tag_content(tag)
+          tag.to_s.delete('|,')
+        end
+
+        def dd_tags(env = ENV)
+          return {} unless dd_tags = env['DD_TAGS']
+
+          to_tags_hash(dd_tags.split(','))
+        end
+
+        def default_tags(env = ENV)
+          dd_tags(env).tap do |tags|
+            tags['dd.internal.entity_id'] = env['DD_ENTITY_ID'] if env.key?('DD_ENTITY_ID')
+            tags['env'] = env['DD_ENV'] if env.key?('DD_ENV')
+            tags['service'] = env['DD_SERVICE'] if env.key?('DD_SERVICE')
+            tags['version'] = env['DD_VERSION'] if env.key?('DD_VERSION')
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/statsd/serialization.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Datadog
+  class Statsd
+    module Serialization
+    end
+  end
+end
+
+require_relative 'serialization/tag_serializer'
+require_relative 'serialization/service_check_serializer'
+require_relative 'serialization/event_serializer'
+require_relative 'serialization/stat_serializer'
+
+require_relative 'serialization/serializer'

data/lib/datadog/statsd/single_thread_sender.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Datadog
+  class Statsd
+    # The SingleThreadSender is a sender synchronously buffering messages
+    # in a `MessageBuffer`.
+    # It is using current Process.PID to check it is the result of a recent fork
+    # and it is reseting the MessageBuffer if that's the case.
+    class SingleThreadSender
+      def initialize(message_buffer, logger: nil, flush_interval: nil)
+        @message_buffer = message_buffer
+        @logger = logger
+        @mx = Mutex.new
+        if flush_interval
+          @flush_timer = Datadog::Statsd::Timer.new(flush_interval) { flush }
+        end
+        # store the pid for which this sender has been created
+        update_fork_pid
+      end
+
+      def add(message)
+        @mx.synchronize {
+          # we have just forked, meaning we have messages in the buffer that we should
+          # not send, they belong to the parent process, let's clear the buffer.
+          if forked?
+            @message_buffer.reset
+            @flush_timer.start if @flush_timer && @flush_timer.stop?
+            update_fork_pid
+          end
+          @message_buffer.add(message)
+        }
+      end
+
+      def flush(*)
+        @mx.synchronize {
+          @message_buffer.flush()
+        }
+      end
+
+      def start()
+        @flush_timer.start if @flush_timer
+      end
+
+      def stop()
+        @flush_timer.stop if @flush_timer
+      end
+
+      # Compatibility with `Sender`
+      def rendez_vous()
+      end
+
+      private
+
+      # below are "fork management" methods to be able to clean the MessageBuffer
+      # if it detects that it is running in a unknown PID.
+
+      def forked?
+        Process.pid != @fork_pid
+      end
+
+      def update_fork_pid
+        @fork_pid = Process.pid
+      end
+    end
+  end
+end

data/lib/datadog/statsd/telemetry.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+require 'time'
+
+module Datadog
+  class Statsd
+    class Telemetry
+      attr_reader :metrics
+      attr_reader :events
+      attr_reader :service_checks
+      attr_reader :bytes_sent
+      attr_reader :bytes_dropped
+      attr_reader :bytes_dropped_queue
+      attr_reader :bytes_dropped_writer
+      attr_reader :packets_sent
+      attr_reader :packets_dropped
+      attr_reader :packets_dropped_queue
+      attr_reader :packets_dropped_writer
+
+      # Rough estimation of maximum telemetry message size without tags
+      MAX_TELEMETRY_MESSAGE_SIZE_WT_TAGS = 50 # bytes
+
+      def initialize(flush_interval, global_tags: [], transport_type: :udp)
+        @flush_interval = flush_interval
+        @global_tags = global_tags
+        @transport_type = transport_type
+        reset
+
+        # TODO: Karim: I don't know why but telemetry tags are serialized
+        # before global tags so by refactoring this, I am keeping the same behavior
+        @serialized_tags = Serialization::TagSerializer.new(
+          client: 'ruby',
+          client_version: VERSION,
+          client_transport: transport_type,
+        ).format(global_tags)
+      end
+
+      def would_fit_in?(max_buffer_payload_size)
+        MAX_TELEMETRY_MESSAGE_SIZE_WT_TAGS + serialized_tags.size < max_buffer_payload_size
+      end
+
+      def reset
+        @metrics = 0
+        @events = 0
+        @service_checks = 0
+        @bytes_sent = 0
+        @bytes_dropped = 0
+        @bytes_dropped_queue = 0
+        @bytes_dropped_writer = 0
+        @packets_sent = 0
+        @packets_dropped = 0
+        @packets_dropped_queue = 0
+        @packets_dropped_writer = 0
+        @next_flush_time = now_in_s + @flush_interval
+      end
+
+      def sent(metrics: 0, events: 0, service_checks: 0, bytes: 0, packets: 0)
+        @metrics += metrics
+        @events += events
+        @service_checks += service_checks
+
+        @bytes_sent += bytes
+        @packets_sent += packets
+      end
+
+      def dropped_queue(bytes: 0, packets: 0)
+        @bytes_dropped += bytes
+        @bytes_dropped_queue += bytes
+        @packets_dropped += packets
+        @packets_dropped_queue += packets
+      end
+
+      def dropped_writer(bytes: 0, packets: 0)
+        @bytes_dropped += bytes
+        @bytes_dropped_writer += bytes
+        @packets_dropped += packets
+        @packets_dropped_writer += packets
+      end
+
+      def should_flush?
+        @next_flush_time < now_in_s
+      end
+
+      def flush
+        [
+          sprintf(pattern, 'metrics', @metrics),
+          sprintf(pattern, 'events', @events),
+          sprintf(pattern, 'service_checks', @service_checks),
+          sprintf(pattern, 'bytes_sent', @bytes_sent),
+          sprintf(pattern, 'bytes_dropped', @bytes_dropped),
+          sprintf(pattern, 'bytes_dropped_queue', @bytes_dropped_queue),
+          sprintf(pattern, 'bytes_dropped_writer', @bytes_dropped_writer),
+          sprintf(pattern, 'packets_sent', @packets_sent),
+          sprintf(pattern, 'packets_dropped', @packets_dropped),
+          sprintf(pattern, 'packets_dropped_queue', @packets_dropped_queue),
+          sprintf(pattern, 'packets_dropped_writer', @packets_dropped_writer),
+        ]
+      end
+
+      private
+      attr_reader :serialized_tags
+
+      def pattern
+        @pattern ||= "datadog.dogstatsd.client.%s:%d|#{COUNTER_TYPE}|##{serialized_tags}"
+      end
+
+      if Kernel.const_defined?('Process') && Process.respond_to?(:clock_gettime)
+        def now_in_s
+          Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
+        end
+      else
+        def now_in_s
+          Time.now.to_i
+        end
+      end
+    end
+  end
+end