statsd-instrument 3.7.0 → 3.9.0

data/lib/statsd/instrument/environment.rb

@@ -82,20 +82,47 @@ module StatsD
         env.fetch("STATSD_ADDR", "localhost:8125")
       end
 
+      def statsd_socket_path
+        env.fetch("STATSD_SOCKET_PATH", "")
+      end
+
       def statsd_default_tags
         env.key?("STATSD_DEFAULT_TAGS") ? env.fetch("STATSD_DEFAULT_TAGS").split(",") : nil
       end
 
       def statsd_buffer_capacity
-        Integer(env.fetch("STATSD_BUFFER_CAPACITY", StatsD::Instrument::BatchedUDPSink::DEFAULT_BUFFER_CAPACITY))
+        Integer(env.fetch("STATSD_BUFFER_CAPACITY", StatsD::Instrument::BatchedSink::DEFAULT_BUFFER_CAPACITY))
       end
 
       def statsd_batching?
         statsd_buffer_capacity > 0 && Float(env.fetch("STATSD_FLUSH_INTERVAL", 1.0)) > 0.0
       end
 
+      def statsd_uds_send?
+        !statsd_socket_path.empty?
+      end
+
       def statsd_max_packet_size
-        Float(env.fetch("STATSD_MAX_PACKET_SIZE", StatsD::Instrument::BatchedUDPSink::DEFAULT_MAX_PACKET_SIZE))
+        if statsd_uds_send?
+          return Float(env.fetch("STATSD_MAX_PACKET_SIZE", StatsD::Instrument::UdsConnection::DEFAULT_MAX_PACKET_SIZE))
+        end
+
+        Float(env.fetch("STATSD_MAX_PACKET_SIZE", StatsD::Instrument::UdpConnection::DEFAULT_MAX_PACKET_SIZE))
+      end
+
+      def statsd_batch_statistics_interval
+        Integer(env.fetch(
+          "STATSD_BATCH_STATISTICS_INTERVAL",
+          StatsD::Instrument::BatchedSink::DEFAULT_STATISTICS_INTERVAL,
+        ))
+      end
+
+      def experimental_aggregation_enabled?
+        env.key?("STATSD_ENABLE_AGGREGATION")
+      end
+
+      def aggregation_interval
+        Float(env.fetch("STATSD_AGGREGATION_INTERVAL", 2.0))
       end
 
       def client
@@ -105,15 +132,24 @@ module StatsD
       def default_sink_for_environment
         case environment
         when "production", "staging"
+          connection = if statsd_uds_send?
+            StatsD::Instrument::UdsConnection.new(statsd_socket_path)
+          else
+            host, port = statsd_addr.split(":")
+            StatsD::Instrument::UdpConnection.new(host, port.to_i)
+          end
+
+          sink = StatsD::Instrument::Sink.new(connection)
           if statsd_batching?
-            StatsD::Instrument::BatchedUDPSink.for_addr(
-              statsd_addr,
+            # if we are batching, wrap the sink in a batched sink
+            return StatsD::Instrument::BatchedSink.new(
+              sink,
               buffer_capacity: statsd_buffer_capacity,
               max_packet_size: statsd_max_packet_size,
+              statistics_interval: statsd_batch_statistics_interval,
             )
-          else
-            StatsD::Instrument::UDPSink.for_addr(statsd_addr)
           end
+          sink
         when "test"
           StatsD::Instrument::NullSink.new
         else

data/lib/statsd/instrument/{udp_sink.rb → sink.rb}

@@ -2,18 +2,21 @@
 
 module StatsD
   module Instrument
-    # @note This class is part of the new Client implementation that is intended
-    #   to become the new default in the next major release of this library.
-    class UDPSink
+    class Sink
       class << self
         def for_addr(addr)
-          host, port_as_string = addr.split(":", 2)
-          new(host, Integer(port_as_string))
+          # if addr is host:port
+          if addr.include?(":")
+            host, port_as_string = addr.split(":", 2)
+            connection = UdpConnection.new(host, Integer(port_as_string))
+            new(connection)
+          else
+            connection = UdsConnection.new(addr)
+            new(connection)
+          end
         end
       end
 
-      attr_reader :host, :port
-
       FINALIZER = ->(object_id) do
         Thread.list.each do |thread|
           if (store = thread["StatsD::UDPSink"])
@@ -22,10 +25,9 @@ module StatsD
         end
       end
 
-      def initialize(host, port)
+      def initialize(connection = nil)
         ObjectSpace.define_finalizer(self, FINALIZER)
-        @host = host
-        @port = port
+        @connection = connection
       end
 
       def sample?(sample_rate)
@@ -35,12 +37,12 @@ module StatsD
       def <<(datagram)
         retried = false
         begin
-          socket.send(datagram, 0)
+          connection.send_datagram(datagram)
         rescue SocketError, IOError, SystemCallError => error
           StatsD.logger.debug do
-            "[StatsD::Instrument::UDPSink] Resetting connection because of #{error.class}: #{error.message}"
+            "[#{self.class.name}] Resetting connection because of #{error.class}: #{error.message}"
           end
-          invalidate_socket
+          invalidate_connection
           if retried
             StatsD.logger.warn do
               "[#{self.class.name}] Events were dropped because of #{error.class}: #{error.message}"
@@ -53,27 +55,34 @@ module StatsD
         self
       end
 
-      def flush(blocking:)
+      def flush(blocking: false)
         # noop
       end
 
-      private
+      def connection_type
+        connection.class.name
+      end
 
-      def invalidate_socket
-        socket = thread_store.delete(object_id)
-        socket&.close
+      def connection
+        thread_store[object_id] ||= @connection
       end
 
-      def socket
-        thread_store[object_id] ||= begin
-          socket = UDPSocket.new
-          socket.connect(@host, @port)
-          socket
-        end
+      def host
+        connection.host
+      end
+
+      def port
+        connection.port
+      end
+
+      private
+
+      def invalidate_connection
+        connection&.close
       end
 
       def thread_store
-        Thread.current["StatsD::UDPSink"] ||= {}
+        Thread.current["StatsD::Sink"] ||= {}
       end
     end
   end

data/lib/statsd/instrument/udp_connection.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module StatsD
+  module Instrument
+    class UdpConnection
+      DEFAULT_MAX_PACKET_SIZE = 1_472
+
+      attr_reader :host, :port
+
+      def initialize(host, port)
+        @host = host
+        @port = port
+      end
+
+      def send_datagram(message)
+        socket.send(message, 0)
+      end
+
+      def close
+        @socket&.close
+        @socket = nil
+      end
+
+      def type
+        :udp
+      end
+
+      private
+
+      def socket
+        @socket ||= begin
+          socket = UDPSocket.new
+          socket.connect(@host, @port)
+          socket
+        end
+      end
+    end
+  end
+end

data/lib/statsd/instrument/uds_connection.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module StatsD
+  module Instrument
+    class UdsConnection
+      DEFAULT_MAX_PACKET_SIZE = 8_192
+
+      def initialize(socket_path, max_packet_size: DEFAULT_MAX_PACKET_SIZE)
+        if max_packet_size <= 0
+          StatsD.logger.warn do
+            "[StatsD::Instrument::UdsConnection] max_packet_size must be greater than 0, " \
+              "using default: #{DEFAULT_MAX_PACKET_SIZE}"
+          end
+        end
+        @socket_path = socket_path
+        @max_packet_size = max_packet_size
+      end
+
+      def send_datagram(message)
+        socket.sendmsg(message, 0)
+      end
+
+      def close
+        @socket&.close
+        @socket = nil
+      end
+
+      def host
+        nil
+      end
+
+      def port
+        nil
+      end
+
+      def type
+        :uds
+      end
+
+      private
+
+      def socket
+        @socket ||= begin
+          socket = Socket.new(Socket::AF_UNIX, Socket::SOCK_DGRAM)
+          socket.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDBUF, @max_packet_size.to_i)
+          socket.connect(Socket.pack_sockaddr_un(@socket_path))
+          socket
+        end
+      end
+    end
+  end
+end

data/lib/statsd/instrument/version.rb

@@ -2,6 +2,6 @@
 
 module StatsD
   module Instrument
-    VERSION = "3.7.0"
+    VERSION = "3.9.0"
   end
 end

data/lib/statsd/instrument.rb

@@ -201,7 +201,10 @@ module StatsD
     #
     # @param method (see #statsd_measure)
     # @param name (see #statsd_measure)
-    # @param metric_options (see #statsd_measure)
+    # @param sample_rate
+    # @param tags
+    # @param no_prefix
+    # @param client
     # @return [void]
     def statsd_count(method, name, sample_rate: nil, tags: nil, no_prefix: false, client: nil)
       add_to_method(method, name, :count) do
@@ -386,19 +389,22 @@ end
 require "statsd/instrument/version"
 require "statsd/instrument/client"
 require "statsd/instrument/datagram"
+require "statsd/instrument/aggregator"
 require "statsd/instrument/dogstatsd_datagram"
 require "statsd/instrument/datagram_builder"
 require "statsd/instrument/statsd_datagram_builder"
 require "statsd/instrument/dogstatsd_datagram_builder"
 require "statsd/instrument/null_sink"
-require "statsd/instrument/udp_sink"
-require "statsd/instrument/batched_udp_sink"
 require "statsd/instrument/capture_sink"
 require "statsd/instrument/log_sink"
 require "statsd/instrument/environment"
 require "statsd/instrument/helpers"
 require "statsd/instrument/assertions"
 require "statsd/instrument/expectation"
+require "statsd/instrument/uds_connection"
+require "statsd/instrument/udp_connection"
+require "statsd/instrument/sink"
+require "statsd/instrument/batched_sink"
 require "statsd/instrument/matchers" if defined?(RSpec)
 require "statsd/instrument/railtie" if defined?(Rails::Railtie)
 require "statsd/instrument/strict" if ENV["STATSD_STRICT_MODE"]

data/statsd-instrument.gemspec

@@ -20,5 +20,7 @@ Gem::Specification.new do |spec|
   spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
+  spec.required_ruby_version = ">= 2.6.0"
+
   spec.metadata['allowed_push_host'] = "https://rubygems.org"
 end

data/test/aggregator_test.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class AggregatorTest < Minitest::Test
+  def setup
+    @sink = StatsD::Instrument::CaptureSink.new(parent: StatsD::Instrument::NullSink.new)
+    @subject = StatsD::Instrument::Aggregator.new(
+      @sink, StatsD::Instrument::DatagramBuilder, nil, [], flush_interval: 0.1
+    )
+  end
+
+  def teardown
+    @sink.clear
+  end
+
+  def test_increment_simple
+    @subject.increment("foo", 1, tags: { foo: "bar" })
+    @subject.increment("foo", 1, tags: { foo: "bar" })
+    @subject.flush
+
+    datagram = @sink.datagrams.first
+    assert_equal("foo", datagram.name)
+    assert_equal(2, datagram.value)
+    assert_equal(1.0, datagram.sample_rate)
+    assert_equal(["foo:bar"], datagram.tags)
+  end
+
+  def test_distribution_simple
+    @subject.aggregate_timing("foo", 1, tags: { foo: "bar" })
+    @subject.aggregate_timing("foo", 100, tags: { foo: "bar" })
+    @subject.flush
+
+    datagram = @sink.datagrams.first
+    assert_equal("foo", datagram.name)
+    assert_equal(2, datagram.value.size)
+    assert_equal([1.0, 100.0], datagram.value)
+  end
+
+  def test_mixed_type_timings
+    @subject.aggregate_timing("foo_ms", 1, tags: { foo: "bar" }, type: :ms)
+    @subject.aggregate_timing("foo_ms", 100, tags: { foo: "bar" }, type: :ms)
+
+    @subject.aggregate_timing("foo_d", 100, tags: { foo: "bar" }, type: :d)
+    @subject.aggregate_timing("foo_d", 120, tags: { foo: "bar" }, type: :d)
+
+    @subject.flush
+
+    assert_equal(2, @sink.datagrams.size)
+    assert_equal(1, @sink.datagrams.filter { |d| d.name == "foo_ms" }.size)
+    assert_equal(1, @sink.datagrams.filter { |d| d.name == "foo_d" }.size)
+    assert_equal("ms", @sink.datagrams.find { |d| d.name == "foo_ms" }.type.to_s)
+    assert_equal("d", @sink.datagrams.find { |d| d.name == "foo_d" }.type.to_s)
+  end
+
+  def test_gauge_simple
+    @subject.gauge("foo", 1, tags: { foo: "bar" })
+    @subject.gauge("foo", 100, tags: { foo: "bar" })
+    @subject.flush
+
+    datagram = @sink.datagrams.first
+    assert_equal("foo", datagram.name)
+    assert_equal(100, datagram.value)
+    assert_equal(:g, datagram.type)
+  end
+
+  def test_increment_with_tags_in_different_orders
+    @subject.increment("foo", 1, tags: ["tag1:val1", "tag2:val2"])
+    @subject.increment("foo", 1, tags: ["tag2:val2", "tag1:val1"])
+    @subject.flush
+
+    assert_equal(2, @sink.datagrams.first.value)
+  end
+
+  def test_increment_with_different_tag_values
+    @subject.increment("foo", 1, tags: ["tag1:val1", "tag2:val2"])
+    @subject.increment("foo", 1, tags: { tag1: "val1", tag2: "val2" })
+
+    @subject.increment("bar")
+    @subject.flush
+
+    assert_equal(2, @sink.datagrams.first.value)
+    assert_equal(2, @sink.datagrams.size)
+    assert_equal(["tag1:val1", "tag2:val2"], @sink.datagrams.first.tags)
+  end
+
+  def test_increment_with_different_metric_names
+    @subject.increment("foo", 1, tags: ["tag1:val1", "tag2:val2"])
+    @subject.increment("bar", 1, tags: ["tag1:val1", "tag2:val2"])
+    @subject.flush
+
+    assert_equal(1, @sink.datagrams.find { |d| d.name == "foo" }.value)
+    assert_equal(1, @sink.datagrams.find { |d| d.name == "bar" }.value)
+  end
+
+  def test_increment_with_different_values
+    @subject.increment("foo", 1, tags: ["tag1:val1", "tag2:val2"])
+    @subject.increment("foo", 2, tags: ["tag1:val1", "tag2:val2"])
+    @subject.flush
+
+    assert_equal(3, @sink.datagrams.first.value)
+  end
+
+  def test_send_mixed_types_will_pass_through
+    @subject.increment("test_counter", 1, tags: ["tag1:val1", "tag2:val2"])
+    @subject.aggregate_timing("test_counter", 100, tags: ["tag1:val1", "tag2:val2"])
+
+    @subject.gauge("test_gauge", 100, tags: ["tag1:val1", "tag2:val2"])
+    @subject.increment("test_gauge", 1, tags: ["tag1:val1", "tag2:val2"])
+
+    @subject.aggregate_timing("test_timing", 100, tags: ["tag1:val1", "tag2:val2"])
+    @subject.gauge("test_timing", 100, tags: ["tag1:val1", "tag2:val2"])
+    @subject.flush
+
+    assert_equal(6, @sink.datagrams.size)
+
+    assert_equal(2, @sink.datagrams.filter { |d| d.name == "test_counter" }.size)
+    assert_equal(2, @sink.datagrams.filter { |d| d.name == "test_gauge" }.size)
+    assert_equal(2, @sink.datagrams.filter { |d| d.name == "test_timing" }.size)
+
+    assert_equal(:d, @sink.datagrams.find { |d| d.name == "test_timing" }.type)
+    assert_equal(:g, @sink.datagrams.find { |d| d.name == "test_gauge" }.type)
+    assert_equal(:c, @sink.datagrams.find { |d| d.name == "test_counter" }.type)
+  end
+
+  def test_with_prefix
+    aggregator = StatsD::Instrument::Aggregator.new(@sink, StatsD::Instrument::DatagramBuilder, "MyApp", [])
+
+    aggregator.increment("foo", 1, tags: ["tag1:val1", "tag2:val2"])
+    aggregator.increment("foo", 1, tags: ["tag1:val1", "tag2:val2"])
+
+    aggregator.increment("foo", 1, tags: ["tag1:val1", "tag2:val2"], no_prefix: true)
+    aggregator.flush
+
+    assert_equal(2, @sink.datagrams.size)
+    assert_equal("MyApp.foo", @sink.datagrams.first.name)
+    assert_equal(2, @sink.datagrams.first.value)
+
+    assert_equal("foo", @sink.datagrams.last.name)
+    assert_equal(1, @sink.datagrams.last.value)
+  end
+end

data/test/client_test.rb

@@ -24,7 +24,7 @@ class ClientTest < Minitest::Test
     assert_equal(["shard:1", "env:production"], client.default_tags)
     assert_equal(StatsD::Instrument::StatsDDatagramBuilder, client.datagram_builder_class)
 
-    assert_kind_of(StatsD::Instrument::BatchedUDPSink, client.sink)
+    assert_kind_of(StatsD::Instrument::BatchedSink, client.sink)
     assert_equal("1.2.3.4", client.sink.host)
     assert_equal(8125, client.sink.port)
   end
@@ -63,6 +63,41 @@ class ClientTest < Minitest::Test
     assert_kind_of(StatsD::Instrument::NullSink, client.sink)
   end
 
+  def test_client_from_env_with_aggregation
+    env = StatsD::Instrument::Environment.new(
+      "STATSD_SAMPLE_RATE" => "0.1",
+      "STATSD_PREFIX" => "foo",
+      "STATSD_DEFAULT_TAGS" => "shard:1,env:production",
+      "STATSD_IMPLEMENTATION" => "statsd",
+      "STATSD_ENABLE_AGGREGATION" => "true",
+      "STATSD_BUFFER_CAPACITY" => "0",
+    )
+    client = StatsD::Instrument::Client.from_env(
+      env,
+      prefix: "bar",
+      implementation: "dogstatsd",
+      sink: StatsD::Instrument::CaptureSink.new(parent: StatsD::Instrument::NullSink.new),
+    )
+
+    assert_equal(0.1, client.default_sample_rate)
+    assert_equal("bar", client.prefix)
+    assert_equal(["shard:1", "env:production"], client.default_tags)
+    assert_equal(StatsD::Instrument::DogStatsDDatagramBuilder, client.datagram_builder_class)
+
+    client.increment("foo", 1, sample_rate: 0.5, tags: { foo: "bar" })
+    client.increment("foo", 1, sample_rate: 0.5, tags: { foo: "bar" })
+
+    client.measure("block_duration_example") { 1 + 1 }
+    client.force_flush
+
+    datagram = client.sink.datagrams.first
+    assert_equal("bar.foo", datagram.name)
+    assert_equal(2, datagram.value)
+
+    datagram = client.sink.datagrams.find { |d| d.name == "bar.block_duration_example" }
+    assert_equal(true, !datagram.nil?)
+  end
+
   def test_capture
     inner_datagrams = nil
 

data/test/datagram_builder_test.rb

@@ -121,4 +121,9 @@ class DatagramBuilderTest < Minitest::Test
     datagram = datagram_builder.c("bar", 1, nil, nil)
     assert_equal("bar:1|c|#foo", datagram)
   end
+
+  def test_timing_with_multiple_values
+    dt = @datagram_builder.timing_value_packed("test_distribution", "d", [0, 3, 6, 9, 12, 15, 18, 21, 24, 27], 1, nil)
+    assert_equal("test_distribution:0:3:6:9:12:15:18:21:24:27|d", dt)
+  end
 end

data/test/dispatcher_stats_test.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class DispatcherStatsTest < Minitest::Test
+  include StatsD::Instrument::Assertions
+
+  def test_maybe_flush
+    stats = StatsD::Instrument::BatchedSink::DispatcherStats.new(0, "udp")
+
+    stats.increment_synchronous_sends
+    stats.increment_batched_sends(1, 1, 1)
+
+    expectations = [
+      StatsD::Instrument::Expectation.increment("statsd_instrument.batched_udp_sink.synchronous_sends", 1),
+      StatsD::Instrument::Expectation.increment("statsd_instrument.batched_udp_sink.batched_sends", 1),
+      StatsD::Instrument::Expectation.gauge("statsd_instrument.batched_udp_sink.avg_buffer_length", 1),
+      StatsD::Instrument::Expectation.gauge("statsd_instrument.batched_udp_sink.avg_batched_packet_size", 1),
+      StatsD::Instrument::Expectation.gauge("statsd_instrument.batched_udp_sink.avg_batch_length", 1),
+    ]
+    assert_statsd_expectations(expectations) { stats.maybe_flush! }
+    assert_equal(0, stats.instance_variable_get(:@synchronous_sends))
+    assert_equal(0, stats.instance_variable_get(:@batched_sends))
+    assert_equal(0, stats.instance_variable_get(:@avg_buffer_length))
+    assert_equal(0, stats.instance_variable_get(:@avg_batched_packet_size))
+    assert_equal(0, stats.instance_variable_get(:@avg_batch_length))
+
+    stats = StatsD::Instrument::BatchedSink::DispatcherStats.new(1, :udp)
+    stats.increment_batched_sends(1, 1, 1)
+    assert_no_statsd_calls { stats.maybe_flush! }
+  end
+
+  def test_calculations_are_correct
+    stats = StatsD::Instrument::BatchedSink::DispatcherStats.new(0, :udp)
+
+    5.times { stats.increment_synchronous_sends }
+    assert_equal(5, stats.instance_variable_get(:@synchronous_sends))
+
+    batches = [
+      { buffer_len: 100, packet_size: 1472, batch_len: 10 },
+      { buffer_len: 90,  packet_size: 1300, batch_len: 20 },
+      { buffer_len: 110, packet_size: 1470, batch_len: 8  },
+      { buffer_len: 500, packet_size: 1000, batch_len: 1  },
+      { buffer_len: 100, packet_size: 30,   batch_len: 99 },
+    ]
+    batches.each do |batch|
+      stats.increment_batched_sends(batch[:buffer_len], batch[:packet_size], batch[:batch_len])
+    end
+    assert_equal(batches.length, stats.instance_variable_get(:@batched_sends))
+    assert_equal(
+      batches.map { |b|
+        b[:buffer_len]
+      }.sum / batches.length,
+      stats.instance_variable_get(:@avg_buffer_length),
+    )
+    assert_equal(
+      batches.map { |b|
+        b[:packet_size]
+      }.sum / batches.length,
+      stats.instance_variable_get(:@avg_batched_packet_size),
+    )
+    assert_equal(
+      batches.map { |b|
+        b[:batch_len]
+      }.sum / batches.length,
+      stats.instance_variable_get(:@avg_batch_length),
+    )
+  end
+end

data/test/environment_test.rb

@@ -48,12 +48,12 @@ class EnvironmentTest < Minitest::Test
 
   def test_client_from_env_uses_batched_udp_sink_in_staging_environment
     env = StatsD::Instrument::Environment.new("STATSD_USE_NEW_CLIENT" => "1", "STATSD_ENV" => "staging")
-    assert_kind_of(StatsD::Instrument::BatchedUDPSink, env.client.sink)
+    assert_kind_of(StatsD::Instrument::BatchedSink, env.client.sink)
   end
 
   def test_client_from_env_uses_batched_udp_sink_in_production_environment
     env = StatsD::Instrument::Environment.new("STATSD_USE_NEW_CLIENT" => "1", "STATSD_ENV" => "production")
-    assert_kind_of(StatsD::Instrument::BatchedUDPSink, env.client.sink)
+    assert_kind_of(StatsD::Instrument::BatchedSink, env.client.sink)
   end
 
   def test_client_from_env_uses_regular_udp_sink_when_flush_interval_is_0
@@ -65,7 +65,7 @@ class EnvironmentTest < Minitest::Test
       "STATSD_ENV" => "staging",
       "STATSD_FLUSH_INTERVAL" => "0.0",
     )
-    assert_kind_of(StatsD::Instrument::UDPSink, env.client.sink)
+    assert_kind_of(StatsD::Instrument::Sink, env.client.sink)
   end
 
   def test_client_from_env_uses_regular_udp_sink_when_buffer_capacity_is_0
@@ -74,6 +74,6 @@ class EnvironmentTest < Minitest::Test
       "STATSD_ENV" => "staging",
       "STATSD_BUFFER_CAPACITY" => "0",
     )
-    assert_kind_of(StatsD::Instrument::UDPSink, env.client.sink)
+    assert_kind_of(StatsD::Instrument::Sink, env.client.sink)
   end
 end