dogstatsd-ruby 5.3.2 → 5.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cfde2027b6fb73eee85ed0c612db51df707c2b01bec2266a3f2049acc34990f1
-  data.tar.gz: 5b1bb3263af1cbdde2bb84cf4d92f2e84ad7e0a0b3eaa5ad67f048750c062c46
+  metadata.gz: aa6ffb63549958aaa8ec14103abbd51082bd945e31d20f0148571ee2c90350b3
+  data.tar.gz: ef1361556ae6de5beb22523187c94ffe7285278918f47d671464b52708ae6973
 SHA512:
-  metadata.gz: 2b563e322f2eaff18eeb07f786d6277290c032e94859d89d856a7bc2512a278424ab5ec9f94f8bf477a222e9ef00075ad2e0293b29dbb0021a5b6551f836b3c9
-  data.tar.gz: 787b04cc62a139289784a9b1992a152ca74fafa3d97afa68944dea95121ff6d3af80164b551b5430d50b263f0d0c8f702d22c1450faaaebdbc4bc69f09c331fe
+  metadata.gz: ba586784b291adaf8a2eaa7a5b866f7931f2c887cb372a69a715f51c0c36a34fba1480cb2ecfd8bbf9573052c2839724fb6389cfa81e157a280e0a75ba0b7072
+  data.tar.gz: 7c2210e57a96e50825362dcedeaa686af6497f85941a9b7dab2a8ca67bbb7cec999d4f13015c5aab90b1a21cb824f5d8c3902cf5dd1421b3a54763a501f441b8

data/README.md

@@ -180,6 +180,15 @@ There are three different kinds of messages:
 
 There is also an implicit message which closes the queue which will cause the sender thread to finish processing and exit.
 
+
+```ruby
+statsd = Datadog::Statsd.new('localhost', 8125)
+```
+
+The message queue's maximum size (in messages) is given by the `sender_queue_size` argument, and has appropriate defaults for UDP (2048) and UDS (512).
+
+The `buffer_flush_interval`, if enabled, is implemented with an additional thread which manages the timing of those flushes.  This additional thread is used even if `single_thread: true`.
+
 ### Usual workflow
 
 You push metrics to the statsd client which writes them quickly to the sender message queue. The sender thread receives those message, buffers them and flushes them to the connection when the buffer limit is reached.

data/lib/datadog/statsd/connection.rb

@@ -38,7 +38,7 @@ module Datadog
           end
         end
 
-        telemetry.dropped(packets: 1, bytes: payload.length) if telemetry
+        telemetry.dropped_writer(packets: 1, bytes: payload.length) if telemetry
         logger.error { "Statsd: #{boom.class} #{boom}" } if logger
         nil
       end

data/lib/datadog/statsd/connection_cfg.rb

@@ -0,0 +1,76 @@
+module Datadog
+  class Statsd
+    class ConnectionCfg
+      attr_reader :host
+      attr_reader :port
+      attr_reader :socket_path
+      attr_reader :transport_type
+
+      def initialize(host: nil, port: nil, socket_path: nil)
+        initialize_with_constructor_args(host: host, port: port, socket_path: socket_path) ||
+          initialize_with_env_vars ||
+          initialize_with_defaults
+      end
+
+      def make_connection(**params)
+        case @transport_type
+        when :udp
+          UDPConnection.new(@host, @port, **params)
+        when :uds
+          UDSConnection.new(@socket_path, **params)
+        end
+      end
+
+      private
+
+      DEFAULT_HOST = '127.0.0.1'
+      DEFAULT_PORT = 8125
+
+      def initialize_with_constructor_args(host: nil, port: nil, socket_path: nil)
+        try_initialize_with(host: host, port: port, socket_path: socket_path,
+          not_both_error_message: 
+            "Both UDP: (host/port #{host}:#{port}) and UDS (socket_path #{socket_path}) " +
+            "constructor arguments were given. Use only one or the other.",
+          )
+      end
+
+      def initialize_with_env_vars()
+        try_initialize_with(
+          host: ENV['DD_AGENT_HOST'],
+          port: ENV['DD_DOGSTATSD_PORT'] && ENV['DD_DOGSTATSD_PORT'].to_i,
+          socket_path: ENV['DD_DOGSTATSD_SOCKET'],
+          not_both_error_message:
+            "Both UDP (DD_AGENT_HOST/DD_DOGSTATSD_PORT #{ENV['DD_AGENT_HOST']}:#{ENV['DD_DOGSTATSD_PORT']}) " +
+            "and UDS (DD_DOGSTATSD_SOCKET #{ENV['DD_DOGSTATSD_SOCKET']}) environment variables are set. " +
+            "Set only one or the other." %
+            [ENV['DD_AGENT_HOST'], ENV['DD_DOGSTATSD_PORT'], ENV['DD_DOGSTATSD_SOCKET']])
+      end
+
+      def initialize_with_defaults()
+        try_initialize_with(host: DEFAULT_HOST, port: DEFAULT_PORT)
+      end
+
+      def try_initialize_with(host: nil, port: nil, socket_path: nil, not_both_error_message: "")
+        if (host || port) && socket_path
+          raise ArgumentError, not_both_error_message
+        end
+
+        if host || port 
+          @host = host || DEFAULT_HOST
+          @port = port || DEFAULT_PORT
+          @socket_path = nil
+          @transport_type = :udp
+          return true
+        elsif socket_path
+          @host = nil
+          @port = nil
+          @socket_path = socket_path
+          @transport_type = :uds
+          return true
+        end
+
+        return false
+      end
+    end
+  end
+end

data/lib/datadog/statsd/forwarder.rb

@@ -7,13 +7,14 @@ module Datadog
       attr_reader :transport_type
 
       def initialize(
-        host: nil,
-        port: nil,
-        socket_path: nil,
+        connection_cfg: nil,
 
         buffer_max_payload_size: nil,
         buffer_max_pool_size: nil,
         buffer_overflowing_stategy: :drop,
+        buffer_flush_interval: nil,
+
+        sender_queue_size: nil,
 
         telemetry_flush_interval: nil,
         global_tags: [],
@@ -22,24 +23,20 @@ module Datadog
 
         logger: nil
       )
-        @transport_type = socket_path.nil? ? :udp : :uds
+        @transport_type = connection_cfg.transport_type
 
         if telemetry_flush_interval
           @telemetry = Telemetry.new(telemetry_flush_interval,
             global_tags: global_tags,
-            transport_type: transport_type
+            transport_type: @transport_type
           )
         end
 
-        @connection = case transport_type
-                      when :udp
-                        UDPConnection.new(host, port, logger: logger, telemetry: telemetry)
-                      when :uds
-                        UDSConnection.new(socket_path, logger: logger, telemetry: telemetry)
-                      end
+        @connection = connection_cfg.make_connection(logger: logger, telemetry: telemetry)
 
         # Initialize buffer
-        buffer_max_payload_size ||= (transport_type == :udp ? UDP_DEFAULT_BUFFER_SIZE : UDS_DEFAULT_BUFFER_SIZE)
+        buffer_max_payload_size ||= (@transport_type == :udp ?
+                                     UDP_DEFAULT_BUFFER_SIZE : UDS_DEFAULT_BUFFER_SIZE)
 
         if buffer_max_payload_size <= 0
           raise ArgumentError, 'buffer_max_payload_size cannot be <= 0'
@@ -54,7 +51,21 @@ module Datadog
           max_pool_size: buffer_max_pool_size || DEFAULT_BUFFER_POOL_SIZE,
           overflowing_stategy: buffer_overflowing_stategy,
         )
-        @sender = (single_thread ? SingleThreadSender : Sender).new(buffer, logger: logger)
+
+        sender_queue_size ||= (@transport_type == :udp ?
+                               UDP_DEFAULT_SENDER_QUEUE_SIZE : UDS_DEFAULT_SENDER_QUEUE_SIZE)
+
+        @sender = single_thread ?
+          SingleThreadSender.new(
+            buffer,
+            logger: logger,
+            flush_interval: buffer_flush_interval) :
+          Sender.new(
+            buffer,
+            logger: logger,
+            flush_interval: buffer_flush_interval,
+            telemetry: @telemetry,
+            queue_size: sender_queue_size)
         @sender.start
       end
 

data/lib/datadog/statsd/sender.rb

@@ -12,10 +12,17 @@ module Datadog
     class Sender
       CLOSEABLE_QUEUES = Queue.instance_methods.include?(:close)
 
-      def initialize(message_buffer, logger: nil)
+      def initialize(message_buffer, telemetry: nil, queue_size: UDP_DEFAULT_BUFFER_SIZE, logger: nil, flush_interval: nil, queue_class: Queue, thread_class: Thread)
         @message_buffer = message_buffer
+        @telemetry = telemetry
+        @queue_size = queue_size
         @logger = logger
         @mx = Mutex.new
+        @queue_class = queue_class
+        @thread_class = thread_class
+        if flush_interval
+          @flush_timer = Datadog::Statsd::Timer.new(flush_interval) { flush(sync: true) }
+        end
       end
 
       def flush(sync: false)
@@ -42,7 +49,7 @@ module Datadog
         return unless message_queue
 
         # Initialize and get the thread's sync queue
-        queue = (Thread.current[:statsd_sync_queue] ||= Queue.new)
+        queue = (@thread_class.current[:statsd_sync_queue] ||= @queue_class.new)
         # tell sender-thread to notify us in the current
         # thread's queue
         message_queue.push(queue)
@@ -68,19 +75,26 @@ module Datadog
             @message_queue = nil
             message_buffer.reset
             start
+            @flush_timer.start if @flush_timer && @flush_timer.stop?
           }
         end
 
-        message_queue << message
+        if message_queue.length <= @queue_size
+          message_queue << message
+        else
+          @telemetry.dropped_queue(packets: 1, bytes: message.bytesize) if @telemetry
+        end
       end
 
       def start
         raise ArgumentError, 'Sender already started' if message_queue
 
         # initialize a new message queue for the background thread
-        @message_queue = Queue.new
+        @message_queue = @queue_class.new
         # start background thread
-        @sender_thread = Thread.new(&method(:send_loop))
+        @sender_thread = @thread_class.new(&method(:send_loop))
+        @sender_thread.name = "Statsd Sender" unless Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.3')
+        @flush_timer.start if @flush_timer
       end
 
       if CLOSEABLE_QUEUES
@@ -91,6 +105,7 @@ module Datadog
           message_queue = @message_queue
           message_queue.close if message_queue
 
+          @flush_timer.stop if @flush_timer
           sender_thread = @sender_thread
           sender_thread.join if sender_thread && join_worker
         end
@@ -102,6 +117,7 @@ module Datadog
           message_queue = @message_queue
           message_queue << :close if message_queue
 
+          @flush_timer.stop if @flush_timer
           sender_thread = @sender_thread
           sender_thread.join if sender_thread && join_worker
         end
@@ -123,7 +139,7 @@ module Datadog
             case message
             when :flush
               message_buffer.flush
-            when Queue
+            when @queue_class
               message.push(:go_on)
             else
               message_buffer.add(message)
@@ -145,7 +161,7 @@ module Datadog
               break
             when :flush
               message_buffer.flush
-            when Queue
+            when @queue_class
               message.push(:go_on)
             else
               message_buffer.add(message)

data/lib/datadog/statsd/single_thread_sender.rb

@@ -7,10 +7,13 @@ module Datadog
     # It is using current Process.PID to check it is the result of a recent fork
     # and it is reseting the MessageBuffer if that's the case.
     class SingleThreadSender
-      def initialize(message_buffer, logger: nil)
+      def initialize(message_buffer, logger: nil, flush_interval: nil)
         @message_buffer = message_buffer
         @logger = logger
         @mx = Mutex.new
+        if flush_interval
+          @flush_timer = Datadog::Statsd::Timer.new(flush_interval) { flush }
+        end
         # store the pid for which this sender has been created
         update_fork_pid
       end
@@ -21,6 +24,7 @@ module Datadog
           # not send, they belong to the parent process, let's clear the buffer.
           if forked?
             @message_buffer.reset
+            @flush_timer.start if @flush_timer && @flush_timer.stop?
             update_fork_pid
           end
           @message_buffer.add(message)
@@ -33,12 +37,12 @@ module Datadog
         }
       end
 
-      # Compatibility with `Sender`
       def start()
+        @flush_timer.start if @flush_timer
       end
 
-      # Compatibility with `Sender`
       def stop()
+        @flush_timer.stop if @flush_timer
       end
 
       # Compatibility with `Sender`

data/lib/datadog/statsd/telemetry.rb

@@ -9,8 +9,12 @@ module Datadog
       attr_reader :service_checks
       attr_reader :bytes_sent
       attr_reader :bytes_dropped
+      attr_reader :bytes_dropped_queue
+      attr_reader :bytes_dropped_writer
       attr_reader :packets_sent
       attr_reader :packets_dropped
+      attr_reader :packets_dropped_queue
+      attr_reader :packets_dropped_writer
 
       # Rough estimation of maximum telemetry message size without tags
       MAX_TELEMETRY_MESSAGE_SIZE_WT_TAGS = 50 # bytes
@@ -40,8 +44,12 @@ module Datadog
         @service_checks = 0
         @bytes_sent = 0
         @bytes_dropped = 0
+        @bytes_dropped_queue = 0
+        @bytes_dropped_writer = 0
         @packets_sent = 0
         @packets_dropped = 0
+        @packets_dropped_queue = 0
+        @packets_dropped_writer = 0
         @next_flush_time = now_in_s + @flush_interval
       end
 
@@ -54,9 +62,18 @@ module Datadog
         @packets_sent += packets
       end
 
-      def dropped(bytes: 0, packets: 0)
+      def dropped_queue(bytes: 0, packets: 0)
         @bytes_dropped += bytes
+        @bytes_dropped_queue += bytes
         @packets_dropped += packets
+        @packets_dropped_queue += packets
+      end
+
+      def dropped_writer(bytes: 0, packets: 0)
+        @bytes_dropped += bytes
+        @bytes_dropped_writer += bytes
+        @packets_dropped += packets
+        @packets_dropped_writer += packets
       end
 
       def should_flush?
@@ -70,8 +87,12 @@ module Datadog
           sprintf(pattern, 'service_checks', @service_checks),
           sprintf(pattern, 'bytes_sent', @bytes_sent),
           sprintf(pattern, 'bytes_dropped', @bytes_dropped),
+          sprintf(pattern, 'bytes_dropped_queue', @bytes_dropped_queue),
+          sprintf(pattern, 'bytes_dropped_writer', @bytes_dropped_writer),
           sprintf(pattern, 'packets_sent', @packets_sent),
           sprintf(pattern, 'packets_dropped', @packets_dropped),
+          sprintf(pattern, 'packets_dropped_queue', @packets_dropped_queue),
+          sprintf(pattern, 'packets_dropped_writer', @packets_dropped_writer),
         ]
       end
 

data/lib/datadog/statsd/timer.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Datadog
+  class Statsd
+    class Timer
+      def initialize(interval, &callback)
+        @mx = Mutex.new
+        @cv = ConditionVariable.new
+        @interval = interval
+        @callback = callback
+        @stop = true
+      end
+
+      def start
+        return unless stop?
+
+        @stop = false
+        @thread = Thread.new do
+          last_execution_time = current_time
+          @mx.synchronize do
+            until @stop
+              timeout = @interval - (current_time - last_execution_time)
+              @cv.wait(@mx, timeout > 0 ? timeout : 0)
+              last_execution_time = current_time
+              @callback.call
+            end
+          end
+        end
+        @thread.name = 'Statsd Timer' unless Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.3')
+      end
+
+      def stop
+        return if @thread.nil?
+
+        @stop = true
+        @mx.synchronize do
+          @cv.signal
+        end
+        @thread.join
+        @thread = nil
+      end
+
+      def stop?
+        @thread.nil? || @thread.stop?
+      end
+
+      private
+
+      if Process.const_defined?(:CLOCK_MONOTONIC)
+        def current_time
+          Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        end
+      else
+        def current_time
+          Time.now
+        end
+      end
+    end
+  end
+end

data/lib/datadog/statsd/udp_connection.rb

@@ -5,20 +5,17 @@ require_relative 'connection'
 module Datadog
   class Statsd
     class UDPConnection < Connection
-      DEFAULT_HOST = '127.0.0.1'
-      DEFAULT_PORT = 8125
-
-      # StatsD host. Defaults to 127.0.0.1.
+      # StatsD host.
       attr_reader :host
 
-      # StatsD port. Defaults to 8125.
+      # StatsD port.
       attr_reader :port
 
       def initialize(host, port, **kwargs)
         super(**kwargs)
 
-        @host = host || ENV.fetch('DD_AGENT_HOST', DEFAULT_HOST)
-        @port = port || ENV.fetch('DD_DOGSTATSD_PORT', DEFAULT_PORT).to_i
+        @host = host
+        @port = port
         @socket = nil
       end
 

data/lib/datadog/statsd/version.rb

@@ -4,6 +4,6 @@ require_relative 'connection'
 
 module Datadog
   class Statsd
-    VERSION = '5.3.2'
+    VERSION = '5.3.3'
   end
 end

data/lib/datadog/statsd.rb

@@ -5,11 +5,13 @@ require_relative 'statsd/version'
 require_relative 'statsd/telemetry'
 require_relative 'statsd/udp_connection'
 require_relative 'statsd/uds_connection'
+require_relative 'statsd/connection_cfg'
 require_relative 'statsd/message_buffer'
 require_relative 'statsd/serialization'
 require_relative 'statsd/sender'
 require_relative 'statsd/single_thread_sender'
 require_relative 'statsd/forwarder'
+require_relative 'statsd/timer'
 
 $deprecation_message_mutex = Mutex.new
 $deprecation_message_done = false
@@ -43,7 +45,12 @@ module Datadog
     UDP_DEFAULT_BUFFER_SIZE = 1_432
     UDS_DEFAULT_BUFFER_SIZE = 8_192
     DEFAULT_BUFFER_POOL_SIZE = Float::INFINITY
+
+    UDP_DEFAULT_SENDER_QUEUE_SIZE = 2048
+    UDS_DEFAULT_SENDER_QUEUE_SIZE = 512
+
     MAX_EVENT_SIZE = 8 * 1_024
+
     # minimum flush interval for the telemetry in seconds
     DEFAULT_TELEMETRY_FLUSH_INTERVAL = 10
 
@@ -72,6 +79,8 @@ module Datadog
     # @option [Logger] logger for debugging
     # @option [Integer] buffer_max_payload_size max bytes to buffer
     # @option [Integer] buffer_max_pool_size max messages to buffer
+    # @option [Integer] sender_queue_size size of the sender queue in number of buffers (multi-thread only)
+    # @option [Numeric] buffer_flush_interval interval in second to flush buffer
     # @option [String] socket_path unix socket path
     # @option [Float] default sample rate if not overridden
     # @option [Boolean] single_thread flushes the metrics on the main thread instead of in a companion thread
@@ -87,6 +96,9 @@ module Datadog
       buffer_max_payload_size: nil,
       buffer_max_pool_size: nil,
       buffer_overflowing_stategy: :drop,
+      buffer_flush_interval: nil,
+
+      sender_queue_size: nil,
 
       logger: nil,
 
@@ -118,9 +130,11 @@ module Datadog
       end
 
       @forwarder = Forwarder.new(
-        host: host,
-        port: port,
-        socket_path: socket_path,
+        connection_cfg: ConnectionCfg.new(
+          host: host,
+          port: port,
+          socket_path: socket_path,
+        ),
 
         global_tags: tags,
         logger: logger,
@@ -130,6 +144,9 @@ module Datadog
         buffer_max_payload_size: buffer_max_payload_size,
         buffer_max_pool_size: buffer_max_pool_size,
         buffer_overflowing_stategy: buffer_overflowing_stategy,
+        buffer_flush_interval: buffer_flush_interval,
+
+        sender_queue_size: sender_queue_size,
 
         telemetry_flush_interval: telemetry_enable ? telemetry_flush_interval : nil,
       )

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dogstatsd-ruby
 version: !ruby/object:Gem::Version
-  version: 5.3.2
+  version: 5.3.3
 platform: ruby
 authors:
 - Rein Henrichs
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-11-03 00:00:00.000000000 Z
+date: 2022-02-02 00:00:00.000000000 Z
 dependencies: []
 description: A Ruby DogStatsd client
 email: code@datadoghq.com
@@ -23,6 +23,7 @@ files:
 - README.md
 - lib/datadog/statsd.rb
 - lib/datadog/statsd/connection.rb
+- lib/datadog/statsd/connection_cfg.rb
 - lib/datadog/statsd/forwarder.rb
 - lib/datadog/statsd/message_buffer.rb
 - lib/datadog/statsd/sender.rb
@@ -34,6 +35,7 @@ files:
 - lib/datadog/statsd/serialization/tag_serializer.rb
 - lib/datadog/statsd/single_thread_sender.rb
 - lib/datadog/statsd/telemetry.rb
+- lib/datadog/statsd/timer.rb
 - lib/datadog/statsd/udp_connection.rb
 - lib/datadog/statsd/uds_connection.rb
 - lib/datadog/statsd/version.rb
@@ -42,9 +44,9 @@ licenses:
 - MIT
 metadata:
   bug_tracker_uri: https://github.com/DataDog/dogstatsd-ruby/issues
-  changelog_uri: https://github.com/DataDog/dogstatsd-ruby/blob/v5.3.2/CHANGELOG.md
-  documentation_uri: https://www.rubydoc.info/gems/dogstatsd-ruby/5.3.2
-  source_code_uri: https://github.com/DataDog/dogstatsd-ruby/tree/v5.3.2
+  changelog_uri: https://github.com/DataDog/dogstatsd-ruby/blob/v5.3.3/CHANGELOG.md
+  documentation_uri: https://www.rubydoc.info/gems/dogstatsd-ruby/5.3.3
+  source_code_uri: https://github.com/DataDog/dogstatsd-ruby/tree/v5.3.3
 post_install_message: |2+
 
   If you are upgrading from v4.x of the dogstatsd-ruby library, note the major change to the threading model: