statsd-instrument 3.7.0 → 3.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c2e01a49d8bb83318bef84628164ab1dec1047a72c1f717969a609f783d3b8be
-  data.tar.gz: bf308ae3bb23958705efcc39c03fa95f99b58bb33df47b7588fe4a7b22d742e9
+  metadata.gz: 7ce05bd8d34026227e2960ccabca96119580dc2d5737fff81a2bdfd8ea18f826
+  data.tar.gz: d619b08700bb735922673013d7e6314f32d76e8744dcfa07966a35595aa27cf4
 SHA512:
-  metadata.gz: 57f0022a72e600871df9f054ad4a003ed2c361287949bfa4af94789460ffa1a9257ffba1f552ac77b786d5c9dc295efb7ccd82edf632f7151d1dda365b282a61
-  data.tar.gz: c458ec4b7177cd746f3fd173641db3127ac97a7683f0f55ff00f84ebe5a81b827d53340566b79ea39adb7327ac967298657d23fcc5f51bd9edced6ea05314beb
+  metadata.gz: 42317b00c680ffc079e89bad712225b8a9656faedf1f8743d9031c26070ae05af0c2ef8a10f1200d94a708023e6cd667d91fbf64a972fecb7d97d327edc31f22
+  data.tar.gz: 540c9a8bccc54633f40e9b2830748e764d6bf5034791bcdacb611be6a7fc615e66a9fa138e2ce58f4a6461f46bb40114fb7507f3f4357d86fccc5d9d6614d244

data/.github/workflows/benchmark.yml

@@ -13,7 +13,6 @@ jobs:
     - name: Set up Ruby
       uses: ruby/setup-ruby@v1
       with:
-        ruby-version: 3.1
         bundler-cache: true
 
     - name: Run benchmark on branch

data/.github/workflows/lint.yml

@@ -13,7 +13,6 @@ jobs:
     - name: Set up Ruby
       uses: ruby/setup-ruby@v1
       with:
-        ruby-version: 2.7
         bundler-cache: true
 
     - name: Run Rubocop

data/.github/workflows/tests.yml

@@ -9,7 +9,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: ['2.6', '2.7', '3.0', '3.1', '3.2', 'ruby-head', 'jruby-9.3.7.0', 'truffleruby-22.2.0']
+        ruby: ['2.6', '2.7', '3.0', '3.1', '3.2', '3.3', 'ruby-head', 'jruby-9.3.7.0', 'truffleruby-22.2.0']
         # Windows on macOS builds started failing, so they are disabled for now
         # platform: [windows-2019, macOS-10.14, ubuntu-18.04]
         # exclude:

data/.rubocop.yml

@@ -5,7 +5,6 @@ require:
   - ./lib/statsd/instrument/rubocop.rb
 
 AllCops:
-  TargetRubyVersion: 2.7
   UseCache: true
   SuggestExtensions: false
   CacheRootDirectory: tmp/rubocop

data/.ruby-version

@@ -0,0 +1 @@
+3.3.0

data/CHANGELOG.md

@@ -6,6 +6,13 @@ section below.
 
 ## Unreleased changes
 
+## Version 3.8.0
+
+- UDP batching will now track statistics about its own batching performance, and
+  emit those statistics to the default sink when `STATSD_BATCH_STATISTICS_INTERVAL`
+  is set to any non-zero value. The default value is zero; additional information
+  on statistics tracked is available in the README.
+
 ## Version 3.7.0
 
 - Add public `.flush` method to sink classes.

data/README.md

@@ -54,6 +54,16 @@ The following environment variables are supported:
   If your network is properly configured to handle larger packets you may try
   to increase this value for better performance, but most network can't handle
   larger packets.
+- `STATSD_BATCH_STATISTICS_INTERVAL`: (default: "0") If non-zero, the `BatchedUDPSink`
+  will track and emit statistics on this interval to the default sink for your environment.
+  The current tracked statistics are:
+
+  - `statsd_instrument.batched_udp_sink.batched_sends`: The number of batches sent, of any size.
+  - `statsd_instrument.batched_udp_sink.synchronous_sends`: The number of times the batched udp sender needed to send a statsd line synchronously, due to the buffer being full.
+  - `statsd_instrument.batched_udp_sink.avg_buffer_length`: The average buffer length, measured at the beginning of each batch.
+  - `statsd_instrument.batched_udp_sink.avg_batched_packet_size`: The average per-batch byte size of the packet sent to the underlying UDPSink.
+  - `statsd_instrument.batched_udp_sink.avg_batch_length`: The average number of statsd lines per batch.
+
 
 ## StatsD keys
 

data/lib/statsd/instrument/batched_udp_sink.rb

@@ -9,6 +9,7 @@ module StatsD
       DEFAULT_BUFFER_CAPACITY = 5_000
       # https://docs.datadoghq.com/developers/dogstatsd/high_throughput/?code-lang=ruby#ensure-proper-packet-sizes
       DEFAULT_MAX_PACKET_SIZE = 1472
+      DEFAULT_STATISTICS_INTERVAL = 0 # in seconds, and 0 implies disabled-by-default.
 
       attr_reader :host, :port
 
@@ -28,7 +29,8 @@ module StatsD
         port,
         thread_priority: DEFAULT_THREAD_PRIORITY,
         buffer_capacity: DEFAULT_BUFFER_CAPACITY,
-        max_packet_size: DEFAULT_MAX_PACKET_SIZE
+        max_packet_size: DEFAULT_MAX_PACKET_SIZE,
+        statistics_interval: DEFAULT_STATISTICS_INTERVAL
       )
         @host = host
         @port = port
@@ -38,6 +40,7 @@ module StatsD
           buffer_capacity,
           thread_priority,
           max_packet_size,
+          statistics_interval,
         )
         ObjectSpace.define_finalizer(self, self.class.finalize(@dispatcher))
       end
@@ -77,8 +80,70 @@ module StatsD
         end
       end
 
+      class DispatcherStats
+        def initialize(interval)
+          # The number of times the batched udp sender needed to
+          # send a statsd line synchronously, due to the buffer
+          # being full.
+          @synchronous_sends = 0
+          # The number of times we send a batch of statsd lines,
+          # of any size.
+          @batched_sends = 0
+          # The average buffer length, measured at the beginning of
+          # each batch.
+          @avg_buffer_length = 0
+          # The average per-batch byte size of the packet sent to
+          # the underlying UDPSink.
+          @avg_batched_packet_size = 0
+          # The average number of statsd lines per batch.
+          @avg_batch_length = 0
+
+          @mutex = Mutex.new
+
+          @interval = interval
+          @since = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        end
+
+        def maybe_flush!(force: false)
+          return if !force && Process.clock_gettime(Process::CLOCK_MONOTONIC) - @since < @interval
+
+          synchronous_sends = 0
+          batched_sends = 0
+          avg_buffer_length = 0
+          avg_batched_packet_size = 0
+          avg_batch_length = 0
+          @mutex.synchronize do
+            synchronous_sends, @synchronous_sends = @synchronous_sends, synchronous_sends
+            batched_sends, @batched_sends = @batched_sends, batched_sends
+            avg_buffer_length, @avg_buffer_length = @avg_buffer_length, avg_buffer_length
+            avg_batched_packet_size, @avg_batched_packet_size = @avg_batched_packet_size, avg_batched_packet_size
+            avg_batch_length, @avg_batch_length = @avg_batch_length, avg_batch_length
+            @since = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          end
+
+          StatsD.increment("statsd_instrument.batched_udp_sink.synchronous_sends", synchronous_sends)
+          StatsD.increment("statsd_instrument.batched_udp_sink.batched_sends", batched_sends)
+          StatsD.gauge("statsd_instrument.batched_udp_sink.avg_buffer_length", avg_buffer_length)
+          StatsD.gauge("statsd_instrument.batched_udp_sink.avg_batched_packet_size", avg_batched_packet_size)
+          StatsD.gauge("statsd_instrument.batched_udp_sink.avg_batch_length", avg_batch_length)
+        end
+
+        def increment_synchronous_sends
+          @mutex.synchronize { @synchronous_sends += 1 }
+        end
+
+        def increment_batched_sends(buffer_len, packet_size, batch_len)
+          @mutex.synchronize do
+            @batched_sends += 1
+            @avg_buffer_length += (buffer_len - @avg_buffer_length) / @batched_sends
+            @avg_batched_packet_size += (packet_size - @avg_batched_packet_size) / @batched_sends
+            @avg_batch_length += (batch_len - @avg_batch_length) / @batched_sends
+          end
+        end
+      end
+
       class Dispatcher
-        def initialize(host, port, buffer_capacity, thread_priority, max_packet_size)
+        def initialize(host, port, buffer_capacity, thread_priority, max_packet_size, statistics_interval)
           @udp_sink = UDPSink.new(host, port)
           @interrupted = false
           @thread_priority = thread_priority
@@ -87,13 +152,18 @@ module StatsD
           @buffer = Buffer.new(buffer_capacity)
           @dispatcher_thread = Thread.new { dispatch }
           @pid = Process.pid
+          if statistics_interval > 0
+            @statistics = DispatcherStats.new(statistics_interval)
+          end
         end
 
         def <<(datagram)
           if !thread_healthcheck || !@buffer.push_nonblock(datagram)
-            # The buffer is full or the thread can't be respaned,
+            # The buffer is full or the thread can't be respawned,
             # we'll send the datagram synchronously
             @udp_sink << datagram
+
+            @statistics&.increment_synchronous_sends
           end
 
           self
@@ -119,6 +189,8 @@ module StatsD
               next_datagram ||= @buffer.pop_nonblock
               break if next_datagram.nil? # no datagram in buffer
             end
+            buffer_len = @buffer.length + 1
+            batch_len = 1
 
             packet << next_datagram
             next_datagram = nil
@@ -126,14 +198,19 @@ module StatsD
               while (next_datagram = @buffer.pop_nonblock)
                 if @max_packet_size - packet.bytesize - 1 > next_datagram.bytesize
                   packet << NEWLINE << next_datagram
+                  batch_len += 1
                 else
                   break
                 end
               end
             end
 
+            packet_size = packet.bytesize
             @udp_sink << packet
             packet.clear
+
+            @statistics&.increment_batched_sends(buffer_len, packet_size, batch_len)
+            @statistics&.maybe_flush!
           end
         end
 

data/lib/statsd/instrument/environment.rb

@@ -98,6 +98,13 @@ module StatsD
         Float(env.fetch("STATSD_MAX_PACKET_SIZE", StatsD::Instrument::BatchedUDPSink::DEFAULT_MAX_PACKET_SIZE))
       end
 
+      def statsd_batch_statistics_interval
+        Integer(env.fetch(
+          "STATSD_BATCH_STATISTICS_INTERVAL",
+          StatsD::Instrument::BatchedUDPSink::DEFAULT_STATISTICS_INTERVAL,
+        ))
+      end
+
       def client
         StatsD::Instrument::Client.from_env(self)
       end
@@ -110,6 +117,7 @@ module StatsD
               statsd_addr,
               buffer_capacity: statsd_buffer_capacity,
               max_packet_size: statsd_max_packet_size,
+              statistics_interval: statsd_batch_statistics_interval,
             )
           else
             StatsD::Instrument::UDPSink.for_addr(statsd_addr)

data/lib/statsd/instrument/version.rb

@@ -2,6 +2,6 @@
 
 module StatsD
   module Instrument
-    VERSION = "3.7.0"
+    VERSION = "3.8.0"
   end
 end

data/statsd-instrument.gemspec

@@ -20,5 +20,7 @@ Gem::Specification.new do |spec|
   spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
+  spec.required_ruby_version = ">= 2.6.0"
+
   spec.metadata['allowed_push_host'] = "https://rubygems.org"
 end

data/test/dispatcher_stats_test.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class DispatcherStatsTest < Minitest::Test
+  include StatsD::Instrument::Assertions
+
+  def test_maybe_flush
+    stats = StatsD::Instrument::BatchedUDPSink::DispatcherStats.new(0)
+
+    stats.increment_synchronous_sends
+    stats.increment_batched_sends(1, 1, 1)
+
+    expectations = [
+      StatsD::Instrument::Expectation.increment("statsd_instrument.batched_udp_sink.synchronous_sends", 1),
+      StatsD::Instrument::Expectation.increment("statsd_instrument.batched_udp_sink.batched_sends", 1),
+      StatsD::Instrument::Expectation.gauge("statsd_instrument.batched_udp_sink.avg_buffer_length", 1),
+      StatsD::Instrument::Expectation.gauge("statsd_instrument.batched_udp_sink.avg_batched_packet_size", 1),
+      StatsD::Instrument::Expectation.gauge("statsd_instrument.batched_udp_sink.avg_batch_length", 1),
+    ]
+    assert_statsd_expectations(expectations) { stats.maybe_flush! }
+    assert_equal(0, stats.instance_variable_get(:@synchronous_sends))
+    assert_equal(0, stats.instance_variable_get(:@batched_sends))
+    assert_equal(0, stats.instance_variable_get(:@avg_buffer_length))
+    assert_equal(0, stats.instance_variable_get(:@avg_batched_packet_size))
+    assert_equal(0, stats.instance_variable_get(:@avg_batch_length))
+
+    stats = StatsD::Instrument::BatchedUDPSink::DispatcherStats.new(1)
+    stats.increment_batched_sends(1, 1, 1)
+    assert_no_statsd_calls { stats.maybe_flush! }
+  end
+
+  def test_calculations_are_correct
+    stats = StatsD::Instrument::BatchedUDPSink::DispatcherStats.new(0)
+
+    5.times { stats.increment_synchronous_sends }
+    assert_equal(5, stats.instance_variable_get(:@synchronous_sends))
+
+    batches = [
+      { buffer_len: 100, packet_size: 1472, batch_len: 10 },
+      { buffer_len: 90,  packet_size: 1300, batch_len: 20 },
+      { buffer_len: 110, packet_size: 1470, batch_len: 8  },
+      { buffer_len: 500, packet_size: 1000, batch_len: 1  },
+      { buffer_len: 100, packet_size: 30,   batch_len: 99 },
+    ]
+    batches.each do |batch|
+      stats.increment_batched_sends(batch[:buffer_len], batch[:packet_size], batch[:batch_len])
+    end
+    assert_equal(batches.length, stats.instance_variable_get(:@batched_sends))
+    assert_equal(
+      batches.map { |b|
+        b[:buffer_len]
+      }.sum / batches.length,
+      stats.instance_variable_get(:@avg_buffer_length),
+    )
+    assert_equal(
+      batches.map { |b|
+        b[:packet_size]
+      }.sum / batches.length,
+      stats.instance_variable_get(:@avg_batched_packet_size),
+    )
+    assert_equal(
+      batches.map { |b|
+        b[:batch_len]
+      }.sum / batches.length,
+      stats.instance_variable_get(:@avg_batch_length),
+    )
+  end
+end

data/test/udp_sink_test.rb

@@ -202,10 +202,31 @@ class BatchedUDPSinkTest < Minitest::Test
     assert(buffer.empty?)
   end
 
+  def test_statistics
+    datagrams = StatsD.singleton_client.capture do
+      buffer_size = 2
+      sink = build_sink(@host, @port, buffer_capacity: buffer_size, statistics_interval: 0.1)
+      2.times { |i| sink << "foo:#{i}|c" }
+      sink.flush(blocking: false)
+      sink.instance_variable_get(:@dispatcher).instance_variable_get(:@statistics).maybe_flush!(force: true)
+    end
+
+    assert(datagrams.any? { |d| d.name.start_with?("statsd_instrument.batched_udp_sink.avg_batch_length") })
+    assert(datagrams.any? { |d| d.name.start_with?("statsd_instrument.batched_udp_sink.avg_batched_packet_size") })
+    assert(datagrams.any? { |d| d.name.start_with?("statsd_instrument.batched_udp_sink.avg_buffer_length") })
+    assert(datagrams.any? { |d| d.name.start_with?("statsd_instrument.batched_udp_sink.batched_sends") })
+    assert(datagrams.any? { |d| d.name.start_with?("statsd_instrument.batched_udp_sink.synchronous_sends") })
+  end
+
   private
 
-  def build_sink(host = @host, port = @port, buffer_capacity: 50)
-    sink = @sink_class.new(host, port, buffer_capacity: buffer_capacity)
+  def build_sink(host = @host, port = @port, buffer_capacity: 50, statistics_interval: 0)
+    sink = @sink_class.new(
+      host,
+      port,
+      buffer_capacity: buffer_capacity,
+      statistics_interval: statistics_interval,
+    )
     @sinks << sink
     sink
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: statsd-instrument
 version: !ruby/object:Gem::Version
-  version: 3.7.0
+  version: 3.8.0
 platform: ruby
 authors:
 - Jesse Storimer
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-03-05 00:00:00.000000000 Z
+date: 2024-06-19 00:00:00.000000000 Z
 dependencies: []
 description: A StatsD client for Ruby apps. Provides metaprogramming methods to inject
   StatsD instrumentation into your code.
@@ -27,6 +27,7 @@ files:
 - ".github/workflows/tests.yml"
 - ".gitignore"
 - ".rubocop.yml"
+- ".ruby-version"
 - ".yardopts"
 - CHANGELOG.md
 - CONTRIBUTING.md
@@ -81,6 +82,7 @@ files:
 - test/client_test.rb
 - test/datagram_builder_test.rb
 - test/datagram_test.rb
+- test/dispatcher_stats_test.rb
 - test/dogstatsd_datagram_builder_test.rb
 - test/environment_test.rb
 - test/helpers/rubocop_helper.rb
@@ -115,14 +117,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 2.6.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.6
+rubygems_version: 3.5.11
 signing_key: 
 specification_version: 4
 summary: A StatsD client for Ruby apps
@@ -136,6 +138,7 @@ test_files:
 - test/client_test.rb
 - test/datagram_builder_test.rb
 - test/datagram_test.rb
+- test/dispatcher_stats_test.rb
 - test/dogstatsd_datagram_builder_test.rb
 - test/environment_test.rb
 - test/helpers/rubocop_helper.rb