statsd-instrument 3.7.0 → 3.9.0

data/test/integration_test.rb

@@ -26,4 +26,55 @@ class IntegrationTest < Minitest::Test
     StatsD.increment("counter")
     assert_equal("counter:1|c", @server.recvfrom(100).first)
   end
+
+  def test_live_local_udp_socket_with_aggregation_flush
+    client = StatsD::Instrument::Environment.new(
+      "STATSD_ADDR" => "#{@server.addr[2]}:#{@server.addr[1]}",
+      "STATSD_IMPLEMENTATION" => "dogstatsd",
+      "STATSD_ENV" => "production",
+      "STATSD_ENABLE_AGGREGATION" => "true",
+      "STATSD_AGGREGATION_INTERVAL" => "5.0",
+    ).client
+
+    10.times do |i|
+      client.increment("counter", 2)
+      client.distribution("test_distribution", 3 * i)
+      client.gauge("test_gauge", 3 * i)
+    end
+
+    client.force_flush
+
+    packets = []
+    while IO.select([@server], nil, nil, 0.1)
+      packets << @server.recvfrom(200).first
+    end
+    packets = packets.map { |packet| packet.split("\n") }.flatten
+
+    assert_equal("counter:20|c", packets.find { |packet| packet.start_with?("counter:") })
+    assert_equal(
+      "test_distribution:0:3:6:9:12:15:18:21:24:27|d",
+      packets.find { |packet| packet.start_with?("test_distribution:") },
+    )
+    assert_equal("test_gauge:27|g", packets.find { |packet| packet.start_with?("test_gauge:") })
+  end
+
+  def test_live_local_udp_socket_with_aggregation_periodic_flush
+    client = StatsD::Instrument::Environment.new(
+      "STATSD_ADDR" => "#{@server.addr[2]}:#{@server.addr[1]}",
+      "STATSD_IMPLEMENTATION" => "dogstatsd",
+      "STATSD_ENV" => "production",
+      "STATSD_ENABLE_AGGREGATION" => "true",
+      "STATSD_AGGREGATION_INTERVAL" => "0.1",
+    ).client
+
+    10.times do
+      client.increment("counter", 2)
+    end
+
+    before_flush = Time.now
+    sleep(0.2)
+
+    assert_equal("counter:20|c", @server.recvfrom(100).first)
+    assert_operator(Time.now - before_flush, :<, 0.3, "Flush and ingest should have happened within 0.4s")
+  end
 end

data/test/test_helper.rb

@@ -6,8 +6,13 @@ end
 
 ENV["ENV"] = "test"
 
+unless ENV.key?("CI")
+  require "minitest/pride"
+end
 require "minitest/autorun"
-require "minitest/pride"
+unless ENV.key?("CI")
+  require "minitest/pride"
+end
 require "mocha/minitest"
 require "statsd-instrument"
 

data/test/udp_sink_test.rb

@@ -107,7 +107,8 @@ module UDPSinkTests
   private
 
   def build_sink(host = @host, port = @port)
-    @sink_class.new(host, port)
+    connection = StatsD::Instrument::UdpConnection.new(host, port)
+    StatsD::Instrument::Sink.new(connection)
   end
 
   def read_datagrams(count, timeout: ENV["CI"] ? 5 : 1)
@@ -132,7 +133,7 @@ class UDPSinkTest < Minitest::Test
     @receiver.bind("localhost", 0)
     @host = @receiver.addr[2]
     @port = @receiver.addr[1]
-    @sink_class = StatsD::Instrument::UDPSink
+    @sink_class = StatsD::Instrument::Sink
   end
 
   def teardown
@@ -167,7 +168,7 @@ class UDPSinkTest < Minitest::Test
     ensure
       StatsD.logger = previous_logger
       # Make sure our fake socket is closed so that it doesn't interfere with other tests
-      udp_sink&.send(:invalidate_socket)
+      udp_sink&.send(:invalidate_connection)
     end
   end
 end
@@ -180,7 +181,7 @@ class BatchedUDPSinkTest < Minitest::Test
     @receiver.bind("localhost", 0)
     @host = @receiver.addr[2]
     @port = @receiver.addr[1]
-    @sink_class = StatsD::Instrument::BatchedUDPSink
+    @sink_class = StatsD::Instrument::BatchedSink
     @sinks = []
   end
 
@@ -202,10 +203,31 @@ class BatchedUDPSinkTest < Minitest::Test
     assert(buffer.empty?)
   end
 
+  def test_statistics
+    datagrams = StatsD.singleton_client.capture do
+      buffer_size = 2
+      sink = build_sink(@host, @port, buffer_capacity: buffer_size, statistics_interval: 0.1)
+      2.times { |i| sink << "foo:#{i}|c" }
+      sink.flush(blocking: false)
+      sink.instance_variable_get(:@dispatcher).instance_variable_get(:@statistics).maybe_flush!(force: true)
+    end
+
+    assert(datagrams.any? { |d| d.name.start_with?("statsd_instrument.batched_udp_sink.avg_batch_length") })
+    assert(datagrams.any? { |d| d.name.start_with?("statsd_instrument.batched_udp_sink.avg_batched_packet_size") })
+    assert(datagrams.any? { |d| d.name.start_with?("statsd_instrument.batched_udp_sink.avg_buffer_length") })
+    assert(datagrams.any? { |d| d.name.start_with?("statsd_instrument.batched_udp_sink.batched_sends") })
+    assert(datagrams.any? { |d| d.name.start_with?("statsd_instrument.batched_udp_sink.synchronous_sends") })
+  end
+
   private
 
-  def build_sink(host = @host, port = @port, buffer_capacity: 50)
-    sink = @sink_class.new(host, port, buffer_capacity: buffer_capacity)
+  def build_sink(host = @host, port = @port, buffer_capacity: 50, statistics_interval: 0)
+    sink = StatsD::Instrument::Sink.for_addr("#{host}:#{port}")
+    sink = @sink_class.new(
+      sink,
+      buffer_capacity: buffer_capacity,
+      statistics_interval: statistics_interval,
+    )
     @sinks << sink
     sink
   end

data/test/uds_sink_test.rb

@@ -0,0 +1,187 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+module UdsTestHelper
+  MAX_READ_BYTES = 64 * 1024
+  private_constant :MAX_READ_BYTES
+
+  private
+
+  def create_socket_file
+    tmpdir = Dir.mktmpdir
+    socket_path = File.join(tmpdir, "sockets", "statsd.sock")
+    FileUtils.mkdir_p(File.dirname(socket_path))
+
+    socket_path
+  end
+
+  def create_receiver(socket_path)
+    FileUtils.rm_f(socket_path)
+    receiver = Socket.new(Socket::AF_UNIX, Socket::SOCK_DGRAM)
+    receiver.setsockopt(Socket::SOL_SOCKET, Socket::SO_REUSEADDR, true)
+    receiver.setsockopt(Socket::SOL_SOCKET, Socket::SO_RCVBUF, (2 * MAX_READ_BYTES).to_i)
+    receiver.bind(Socket.pack_sockaddr_un(socket_path))
+
+    receiver
+  end
+
+  def build_sink(socket_path)
+    connection = StatsD::Instrument::UdsConnection.new(socket_path)
+    @sink_class.new(connection)
+  end
+
+  def sink
+    @sink ||= build_sink(@socket_path)
+  end
+
+  def skip_on_jruby(message = "JRuby does not support UNIX domain sockets")
+    skip(message) if RUBY_PLATFORM == "java"
+  end
+
+  def read_datagrams(count, timeout: ENV["CI"] ? 5 : 1)
+    datagrams = []
+    count.times do
+      if @receiver.wait_readable(timeout)
+
+        datagrams += @receiver.recvfrom(MAX_READ_BYTES).first.lines(chomp: true)
+        break if datagrams.size >= count
+      else
+        break
+      end
+    end
+
+    datagrams
+  end
+end
+
+class UdsSinkTest < Minitest::Test
+  include UdsTestHelper
+
+  def setup
+    @sink_class = StatsD::Instrument::Sink
+    @socket_path = create_socket_file
+    skip_on_jruby
+
+    @receiver = create_receiver(@socket_path)
+  end
+
+  def teardown
+    return if RUBY_PLATFORM == "java"
+
+    @receiver.close
+    FileUtils.rm_f(@socket_path)
+  end
+
+  def test_send_metric_with_tags
+    metric = "test.metric"
+    value = 42
+    tags = { region: "us-west", environment: "production" }
+    sink << "#{metric}:#{value}|c|##{"region:#{tags[:region]},environment:#{tags[:environment]}"}"
+    # Assert that the metric with tags was sent successfully
+
+    datagrams = read_datagrams(1)
+    assert_equal("test.metric:42|c|#region:us-west,environment:production".b, datagrams.first)
+  end
+
+  def test_send_metric_with_sample_rate
+    metric = "test.metric"
+    value = 42
+    sample_rate = 0.5
+    sink << "#{metric}:#{value}|c|@#{sample_rate}"
+    datagrams = read_datagrams(1)
+    assert_equal("test.metric:42|c|@0.5".b, datagrams.first)
+  end
+
+  def test_flush_with_empty_batch
+    sink.flush
+    datagrams = read_datagrams(1, timeout: 0.1)
+    assert_empty(datagrams)
+  end
+end
+
+class BatchedUdsSinkTest < Minitest::Test
+  include UdsTestHelper
+
+  def setup
+    @socket_path = create_socket_file
+    @sink_class = StatsD::Instrument::BatchedSink
+    @sinks = []
+
+    skip_on_jruby
+
+    @receiver = create_receiver(@socket_path)
+  end
+
+  def teardown
+    return if RUBY_PLATFORM == "java"
+
+    @receiver.close
+    FileUtils.remove_entry(@socket_path)
+    @sinks.each(&:shutdown)
+  end
+
+  def test_send_metric_with_tags
+    metric = "test.metric"
+    value = 42
+    tags = { region: "us-west", environment: "production" }
+    sink << "#{metric}:#{value}|c|##{"region:#{tags[:region]},environment:#{tags[:environment]}"}"
+    datagrams = read_datagrams(1)
+    assert_equal("test.metric:42|c|#region:us-west,environment:production".b, datagrams.first)
+  end
+
+  def test_send_metric_with_sample_rate
+    metric = "test.metric"
+    value = 42
+    sample_rate = 0.5
+    sink << "#{metric}:#{value}|c|@#{sample_rate}"
+    datagrams = read_datagrams(1)
+    assert_equal("test.metric:42|c|@0.5".b, datagrams.first)
+  end
+
+  def test_flush_with_empty_batch
+    sink.flush(blocking: false)
+    datagrams = read_datagrams(1, timeout: 0.1)
+    assert_empty(datagrams)
+  end
+
+  def test_flush
+    buffer_size = 50
+    sink = build_sink(@socket_path, buffer_capacity: buffer_size)
+    dispatcher = sink.instance_variable_get(:@dispatcher)
+    buffer = dispatcher.instance_variable_get(:@buffer)
+    (buffer_size * 2).times { |i| sink << "foo:#{i}|c" }
+    assert(!buffer.empty?)
+    sink.flush(blocking: false)
+    assert(buffer.empty?)
+  end
+
+  def test_statistics
+    datagrams = StatsD.singleton_client.capture do
+      buffer_size = 2
+      sink = build_sink(@socket_path, buffer_capacity: buffer_size, statistics_interval: 0.1)
+      2.times { |i| sink << "foo:#{i}|c" }
+      sink.flush(blocking: false)
+      sink.instance_variable_get(:@dispatcher).instance_variable_get(:@statistics).maybe_flush!(force: true)
+    end
+
+    assert(datagrams.any? { |d| d.name.start_with?("statsd_instrument.batched_uds_sink.avg_batch_length") })
+    assert(datagrams.any? { |d| d.name.start_with?("statsd_instrument.batched_uds_sink.avg_batched_packet_size") })
+    assert(datagrams.any? { |d| d.name.start_with?("statsd_instrument.batched_uds_sink.avg_buffer_length") })
+    assert(datagrams.any? { |d| d.name.start_with?("statsd_instrument.batched_uds_sink.batched_sends") })
+    assert(datagrams.any? { |d| d.name.start_with?("statsd_instrument.batched_uds_sink.synchronous_sends") })
+  end
+
+  private
+
+  def build_sink(socket_path, buffer_capacity: 50, statistics_interval: 0)
+    sink = StatsD::Instrument::Sink.for_addr(socket_path)
+    sink = @sink_class.new(
+      sink,
+      buffer_capacity: buffer_capacity,
+      statistics_interval: statistics_interval,
+    )
+    @sinks << sink
+    sink
+  end
+end

metadata

@@ -1,16 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: statsd-instrument
 version: !ruby/object:Gem::Version
-  version: 3.7.0
+  version: 3.9.0
 platform: ruby
 authors:
 - Jesse Storimer
 - Tobias Lutke
 - Willem van Bergen
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-03-05 00:00:00.000000000 Z
+date: 2024-08-19 00:00:00.000000000 Z
 dependencies: []
 description: A StatsD client for Ruby apps. Provides metaprogramming methods to inject
   StatsD instrumentation into your code.
@@ -21,12 +21,14 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".github/CODEOWNERS"
+- ".github/pull_request_template.md"
 - ".github/workflows/benchmark.yml"
 - ".github/workflows/cla.yml"
 - ".github/workflows/lint.yml"
 - ".github/workflows/tests.yml"
 - ".gitignore"
 - ".rubocop.yml"
+- ".ruby-version"
 - ".yardopts"
 - CHANGELOG.md
 - CONTRIBUTING.md
@@ -42,8 +44,9 @@ files:
 - bin/rubocop
 - lib/statsd-instrument.rb
 - lib/statsd/instrument.rb
+- lib/statsd/instrument/aggregator.rb
 - lib/statsd/instrument/assertions.rb
-- lib/statsd/instrument/batched_udp_sink.rb
+- lib/statsd/instrument/batched_sink.rb
 - lib/statsd/instrument/capture_sink.rb
 - lib/statsd/instrument/client.rb
 - lib/statsd/instrument/datagram.rb
@@ -66,12 +69,15 @@ files:
 - lib/statsd/instrument/rubocop/positional_arguments.rb
 - lib/statsd/instrument/rubocop/singleton_configuration.rb
 - lib/statsd/instrument/rubocop/splat_arguments.rb
+- lib/statsd/instrument/sink.rb
 - lib/statsd/instrument/statsd_datagram_builder.rb
 - lib/statsd/instrument/strict.rb
-- lib/statsd/instrument/udp_sink.rb
+- lib/statsd/instrument/udp_connection.rb
+- lib/statsd/instrument/uds_connection.rb
 - lib/statsd/instrument/version.rb
 - shipit.rubygems.yml
 - statsd-instrument.gemspec
+- test/aggregator_test.rb
 - test/assertions_test.rb
 - test/benchmark/clock_gettime.rb
 - test/benchmark/metrics.rb
@@ -81,6 +87,7 @@ files:
 - test/client_test.rb
 - test/datagram_builder_test.rb
 - test/datagram_test.rb
+- test/dispatcher_stats_test.rb
 - test/dogstatsd_datagram_builder_test.rb
 - test/environment_test.rb
 - test/helpers/rubocop_helper.rb
@@ -102,12 +109,13 @@ files:
 - test/statsd_test.rb
 - test/test_helper.rb
 - test/udp_sink_test.rb
+- test/uds_sink_test.rb
 homepage: https://github.com/Shopify/statsd-instrument
 licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -115,18 +123,19 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 2.6.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.6
-signing_key: 
+rubygems_version: 3.5.17
+signing_key:
 specification_version: 4
 summary: A StatsD client for Ruby apps
 test_files:
+- test/aggregator_test.rb
 - test/assertions_test.rb
 - test/benchmark/clock_gettime.rb
 - test/benchmark/metrics.rb
@@ -136,6 +145,7 @@ test_files:
 - test/client_test.rb
 - test/datagram_builder_test.rb
 - test/datagram_test.rb
+- test/dispatcher_stats_test.rb
 - test/dogstatsd_datagram_builder_test.rb
 - test/environment_test.rb
 - test/helpers/rubocop_helper.rb
@@ -157,3 +167,4 @@ test_files:
 - test/statsd_test.rb
 - test/test_helper.rb
 - test/udp_sink_test.rb
+- test/uds_sink_test.rb