statsd-instrument 3.9.7 → 3.9.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f878f6a9d5a1ebb2aa0459eec131628d04a19b85b9cd7bc4b0493f5bc946423f
-  data.tar.gz: 41e2a93dd2257655bc64235aebf4637751101f69da4f342aeaab5a3b56bf21e3
+  metadata.gz: 6aa9b8412d646528092adba3cbfcf173e80c4c9f6a69ce2a49db6c74d74fded6
+  data.tar.gz: 38aaac366290965b067e2e167bffa030aa90d185e595e7c6ed1d687683607bb0
 SHA512:
-  metadata.gz: 4aaf2c5f29bcf2c815298d8047208be603c360d2c3c4cd342fd77c0337f310fffb51c0d3706338d922e0125fdad824084097828910ef422933f2526411904523
-  data.tar.gz: ba4d68b8078cb033ec887f35d8682c7771291b3197a054269aa5e44ab6538f29f0b261e3b8c6a5259e02cf3a7f9aa0376ad21fb7cd709b50fc4b516e7ba6b4cf
+  metadata.gz: 32cbfe82af236e100778e9367ed1c7cf5a1b4c2ed4bed6969d1e319738eddfe192c0e521c2917ac4b4af5ac1e4e36f3f551828fa9fc69a06e9009d015726f465
+  data.tar.gz: 0ac5714732d77dcda2ac89ea92dfd51ae99ed1cc021766eaed26e7b28310181c51c5a095ed545862be513a8b17348811715dff2760f6a9ae58dd431a5337080a

data/.github/workflows/tests.yml

@@ -4,16 +4,12 @@ on: push
 
 jobs:
   test:
-    name: Ruby ${{ matrix.ruby }} on ubuntu-latest
-    runs-on: ubuntu-latest
+    name: Ruby ${{ matrix.ruby }} on ubuntu-22.04
+    runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
       matrix:
-        ruby: ['2.6', '2.7', '3.0', '3.1', '3.2', '3.3', 'ruby-head', 'jruby-9.4.8.0', 'truffleruby-22.3.1']
-        # Windows on macOS builds started failing, so they are disabled for now
-        # platform: [windows-2019, macOS-10.14, ubuntu-18.04]
-        # exclude:
-        # ...
+        ruby: ['2.7', '3.0', '3.1', '3.2', '3.3', 'ruby-head', 'jruby-9.4.9.0', 'truffleruby-22.3.1']
 
     steps:
     - uses: actions/checkout@v4

data/CHANGELOG.md

@@ -6,6 +6,11 @@ section below.
 
 ## Unreleased changes
 
+## Version 3.9.8
+
+- [#390](https://github.com/Shopify/statsd-instrument/pull/391) - Fixing bug in Environment when using UDS. The max packet size option was not being passed to the 
+UDS connection, causing messages that were too large to be dropped (specially sensitive when used together with BatchedSink).
+
 ## Version 3.9.7
 
 - [#389](https://github.com/Shopify/statsd-instrument/pull/389) - Fixing bug with BatchedSink constructor when using UDS, the constructor was not properly passing the Sink to the BatchedSink.

data/Makefile

@@ -0,0 +1,11 @@
+.PHONY: test lint update
+test:
+	bundle exec rake test
+
+lint:
+	bundle exec rake lint_fix
+
+update:
+	bundle update
+
+check: update lint test

data/benchmark/local-udp-throughput

@@ -10,6 +10,24 @@ require "datadog/statsd"
 require "forwardable"
 require "vernier"
 
+module Datadog
+  class Statsd
+    class Telemetry
+      def flush
+        ["bytes", "packets"].each do |kind|
+          ["sent", "dropped", "dropped_queue", "dropped_writer"].each do |measure|
+            var = "#{kind}_#{measure}"
+            puts "#{var}: #{instance_variable_get("@#{var}".to_sym)}"
+          end
+        end
+        puts "-" * 10
+
+        []
+      end
+    end
+  end
+end
+
 class DatadogShim
   extend Forwardable
 
@@ -222,3 +240,40 @@ benchmark_implementation(
 if ENABLE_PROFILING
   Vernier.stop_profile
 end
+
+if ENABLE_PROFILING
+  Vernier.start_profile(out: "tmp/benchmark_profile_datadog.json")
+end
+benchmark_implementation(
+  "Datadog client with UDP and delayed serialization",
+  {
+    buffer_max_payload_size: 4096,
+    delay_serialization: true,
+    telemetry_flush_interval: 1,
+    # The datadog implemtation will drop metrics if the queue is full
+    # to imitate the behavior of the implementation of this gem, we set
+    # the queue size to infinity to avoid dropping metrics.
+    sender_queue_size: Float::INFINITY,
+  },
+  true,
+)
+if ENABLE_PROFILING
+  Vernier.stop_profile
+end
+
+if ENABLE_PROFILING
+  Vernier.start_profile(out: "tmp/benchmark_profile_datadog.json")
+end
+benchmark_implementation(
+  "UDP - Datadog gem - using: delay_serialization, multi-threaded, allow dropping samples",
+  {
+    buffer_max_payload_size: 4096,
+    delay_serialization: true,
+    telemetry_flush_interval: 1,
+    sender_queue_size: 250_000,
+  },
+  true,
+)
+if ENABLE_PROFILING
+  Vernier.stop_profile
+end

data/lib/statsd/instrument/client.rb

@@ -459,7 +459,6 @@ module StatsD
       # @note Supported by the Datadog implementation only.
       def event(title, text, timestamp: nil, hostname: nil, aggregation_key: nil, priority: nil,
         source_type_name: nil, alert_type: nil, tags: nil, no_prefix: false)
-
         emit(datagram_builder(no_prefix: no_prefix)._e(
           title,
           text,

data/lib/statsd/instrument/connection_behavior.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module StatsD
+  module Instrument
+    module ConnectionBehavior
+      def close
+        @socket&.close
+      rescue IOError, SystemCallError => e
+        StatsD.logger.debug do
+          "[#{self.class.name}] Error closing socket: #{e.class}: #{e.message}"
+        end
+      ensure
+        @socket = nil
+      end
+
+      def send_buffer_size
+        if socket
+          send_buffer_size_from_socket(socket)
+        else
+          @max_packet_size
+        end
+      end
+
+      def type
+        raise NotImplementedError, "#{self.class} must implement #type"
+      end
+
+      private
+
+      def send_buffer_size_from_socket(original_socket)
+        original_socket.getsockopt(Socket::SOL_SOCKET, Socket::SO_SNDBUF).int
+      end
+
+      def setup_socket(original_socket)
+        original_socket.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDBUF, @max_packet_size.to_i)
+        if send_buffer_size_from_socket(original_socket) < @max_packet_size
+          StatsD.logger.warn do
+            "[#{self.class.name}] Could not set socket send buffer size to #{@max_packet_size} " \
+              "allowed size by environment/OS is (#{send_buffer_size_from_socket(original_socket)})."
+          end
+        end
+        original_socket
+      rescue IOError => e
+        StatsD.logger.debug do
+          "[#{self.class.name}] Failed to create socket: #{e.class}: #{e.message}"
+        end
+        nil
+      end
+    end
+  end
+end

data/lib/statsd/instrument/dogstatsd_datagram_builder.rb

@@ -33,7 +33,6 @@ module StatsD
       # @see https://docs.datadoghq.com/developers/dogstatsd/datagram_shell/#events
       def _e(title, text, timestamp: nil, hostname: nil, aggregation_key: nil, priority: nil,
         source_type_name: nil, alert_type: nil, tags: nil)
-
         escaped_title = "#{@prefix}#{title}".gsub("\n", '\n')
         escaped_text = text.gsub("\n", '\n')
 

data/lib/statsd/instrument/environment.rb

@@ -104,10 +104,10 @@ module StatsD
 
       def statsd_max_packet_size
         if statsd_uds_send?
-          return Float(env.fetch("STATSD_MAX_PACKET_SIZE", StatsD::Instrument::UdsConnection::DEFAULT_MAX_PACKET_SIZE))
+          Integer(env.fetch("STATSD_MAX_PACKET_SIZE", StatsD::Instrument::UdsConnection::DEFAULT_MAX_PACKET_SIZE))
+        else
+          Integer(env.fetch("STATSD_MAX_PACKET_SIZE", StatsD::Instrument::UdpConnection::DEFAULT_MAX_PACKET_SIZE))
         end
-
-        Float(env.fetch("STATSD_MAX_PACKET_SIZE", StatsD::Instrument::UdpConnection::DEFAULT_MAX_PACKET_SIZE))
       end
 
       def statsd_batch_statistics_interval
@@ -140,19 +140,32 @@ module StatsD
         case environment
         when "production", "staging"
           connection = if statsd_uds_send?
-            StatsD::Instrument::UdsConnection.new(statsd_socket_path)
+            StatsD::Instrument::UdsConnection.new(
+              statsd_socket_path,
+              max_packet_size: statsd_max_packet_size,
+            )
           else
             host, port = statsd_addr.split(":")
-            StatsD::Instrument::UdpConnection.new(host, port.to_i)
+            StatsD::Instrument::UdpConnection.new(
+              host,
+              port.to_i,
+              max_packet_size: statsd_max_packet_size,
+            )
           end
 
           sink = StatsD::Instrument::Sink.new(connection)
           if statsd_batching?
-            # if we are batching, wrap the sink in a batched sink
+            current_send_buffer_size = connection.send_buffer_size
+            if current_send_buffer_size < statsd_max_packet_size
+              StatsD.logger.warn do
+                "[StatsD::Instrument::Environment] Send buffer size #{current_send_buffer_size} differs from " \
+                  "max packet size #{statsd_max_packet_size}. Using send buffer size as max packet size."
+              end
+            end
             return StatsD::Instrument::BatchedSink.new(
               sink,
               buffer_capacity: statsd_buffer_capacity,
-              max_packet_size: statsd_max_packet_size,
+              max_packet_size: [current_send_buffer_size, statsd_max_packet_size].min,
               statistics_interval: statsd_batch_statistics_interval,
             )
           end

data/lib/statsd/instrument/expectation.rb

@@ -36,7 +36,6 @@ module StatsD
 
       def initialize(client: nil, type:, name:, value: nil,
         sample_rate: nil, tags: nil, no_prefix: false, times: 1)
-
         @type = type
         @name = no_prefix ? name : StatsD::Instrument::Helpers.prefix_metric(name, client: client)
         @value = normalized_value_for_type(type, value) if value

data/lib/statsd/instrument/sink.rb

@@ -40,12 +40,15 @@ module StatsD
           connection.send_datagram(datagram)
         rescue SocketError, IOError, SystemCallError => error
           StatsD.logger.debug do
-            "[#{self.class.name}] Resetting connection because of #{error.class}: #{error.message}"
+            "[#{self.class.name}] [#{connection.class.name}] " \
+              "Resetting connection because of #{error.class}: #{error.message}"
           end
           invalidate_connection
           if retried
             StatsD.logger.warn do
-              "[#{self.class.name}] Events were dropped because of #{error.class}: #{error.message}"
+              "[#{self.class.name}] [#{connection.class.name}] " \
+                "Events were dropped (after retrying) because of #{error.class}: #{error.message}. " \
+                "Message size: #{datagram.bytesize} bytes."
             end
           else
             retried = true

data/lib/statsd/instrument/strict.rb

@@ -65,7 +65,6 @@ module StatsD
 
       def event(title, text, tags: nil, no_prefix: false,
         hostname: nil, timestamp: nil, aggregation_key: nil, priority: nil, source_type_name: nil, alert_type: nil)
-
         super
       end
 

data/lib/statsd/instrument/udp_connection.rb

@@ -3,24 +3,22 @@
 module StatsD
   module Instrument
     class UdpConnection
+      include ConnectionBehavior
+
       DEFAULT_MAX_PACKET_SIZE = 1_472
 
       attr_reader :host, :port
 
-      def initialize(host, port)
+      def initialize(host, port, max_packet_size: DEFAULT_MAX_PACKET_SIZE)
         @host = host
         @port = port
+        @max_packet_size = max_packet_size
       end
 
       def send_datagram(message)
         socket.send(message, 0)
       end
 
-      def close
-        @socket&.close
-        @socket = nil
-      end
-
       def type
         :udp
       end
@@ -29,9 +27,10 @@ module StatsD
 
       def socket
         @socket ||= begin
-          socket = UDPSocket.new
-          socket.connect(@host, @port)
-          socket
+          udp_socket = UDPSocket.new
+          setup_socket(udp_socket)&.tap do |s|
+            s.connect(@host, @port)
+          end
         end
       end
     end

data/lib/statsd/instrument/uds_connection.rb

@@ -3,6 +3,8 @@
 module StatsD
   module Instrument
     class UdsConnection
+      include ConnectionBehavior
+
       DEFAULT_MAX_PACKET_SIZE = 8_192
 
       def initialize(socket_path, max_packet_size: DEFAULT_MAX_PACKET_SIZE)
@@ -17,12 +19,7 @@ module StatsD
       end
 
       def send_datagram(message)
-        socket.sendmsg(message, 0)
-      end
-
-      def close
-        @socket&.close
-        @socket = nil
+        socket&.sendmsg(message, 0)
       end
 
       def host
@@ -41,10 +38,10 @@ module StatsD
 
       def socket
         @socket ||= begin
-          socket = Socket.new(Socket::AF_UNIX, Socket::SOCK_DGRAM)
-          socket.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDBUF, @max_packet_size.to_i)
-          socket.connect(Socket.pack_sockaddr_un(@socket_path))
-          socket
+          unix_socket = Socket.new(Socket::AF_UNIX, Socket::SOCK_DGRAM)
+          setup_socket(unix_socket)&.tap do |s|
+            s.connect(Socket.pack_sockaddr_un(@socket_path))
+          end
         end
       end
     end

data/lib/statsd/instrument/version.rb

@@ -2,6 +2,6 @@
 
 module StatsD
   module Instrument
-    VERSION = "3.9.7"
+    VERSION = "3.9.8"
   end
 end

data/lib/statsd/instrument.rb

@@ -401,6 +401,7 @@ require "statsd/instrument/environment"
 require "statsd/instrument/helpers"
 require "statsd/instrument/assertions"
 require "statsd/instrument/expectation"
+require "statsd/instrument/connection_behavior"
 require "statsd/instrument/uds_connection"
 require "statsd/instrument/udp_connection"
 require "statsd/instrument/sink"

data/test/environment_test.rb

@@ -76,4 +76,93 @@ class EnvironmentTest < Minitest::Test
     )
     assert_kind_of(StatsD::Instrument::Sink, env.client.sink)
   end
+
+  def test_client_from_env_uses_uds_sink_with_correct_packet_size_in_production
+    skip_on_jruby("JRuby does not support UNIX domain sockets")
+    socket_path = "/tmp/statsd-test-#{Process.pid}.sock"
+
+    # Create a UDS server socket
+    server = Socket.new(Socket::AF_UNIX, Socket::SOCK_DGRAM)
+    server.bind(Socket.pack_sockaddr_un(socket_path))
+
+    env = StatsD::Instrument::Environment.new(
+      "STATSD_ENV" => "production",
+      "STATSD_SOCKET_PATH" => socket_path,
+      "STATSD_MAX_PACKET_SIZE" => "65507",
+      "STATSD_USE_NEW_CLIENT" => "1",
+    )
+
+    begin
+      client = env.client
+      sink = client.sink
+      connection = sink.connection
+
+      assert_kind_of(StatsD::Instrument::UdsConnection, connection)
+      assert_equal(65507, connection.instance_variable_get(:@max_packet_size))
+    ensure
+      server.close
+      File.unlink(socket_path) if File.exist?(socket_path)
+    end
+  end
+
+  def test_client_from_env_uses_default_packet_size_for_uds_when_not_specified
+    skip_on_jruby("JRuby does not support UNIX domain sockets")
+    socket_path = "/tmp/statsd-test-#{Process.pid}-default.sock"
+
+    # Create a UDS server socket
+    server = Socket.new(Socket::AF_UNIX, Socket::SOCK_DGRAM)
+    server.bind(Socket.pack_sockaddr_un(socket_path))
+
+    env = StatsD::Instrument::Environment.new(
+      "STATSD_ENV" => "production",
+      "STATSD_SOCKET_PATH" => socket_path,
+      "STATSD_USE_NEW_CLIENT" => "1",
+    )
+
+    begin
+      client = env.client
+      sink = client.sink
+      connection = sink.connection
+
+      assert_kind_of(StatsD::Instrument::UdsConnection, connection)
+      assert_equal(
+        StatsD::Instrument::UdsConnection::DEFAULT_MAX_PACKET_SIZE,
+        connection.instance_variable_get(:@max_packet_size),
+      )
+    ensure
+      server.close
+      File.unlink(socket_path) if File.exist?(socket_path)
+    end
+  end
+
+  def test_client_from_env_uses_batched_uds_sink_with_correct_packet_size
+    skip_on_jruby("JRuby does not support UNIX domain sockets")
+    socket_path = "/tmp/statsd-test-#{Process.pid}-batched.sock"
+
+    # Create a UDS server socket
+    server = Socket.new(Socket::AF_UNIX, Socket::SOCK_DGRAM)
+    server.bind(Socket.pack_sockaddr_un(socket_path))
+
+    env = StatsD::Instrument::Environment.new(
+      "STATSD_ENV" => "production",
+      "STATSD_SOCKET_PATH" => socket_path,
+      "STATSD_MAX_PACKET_SIZE" => "65507",
+      "STATSD_BUFFER_CAPACITY" => "1000",
+      "STATSD_USE_NEW_CLIENT" => "1",
+    )
+
+    begin
+      client = env.client
+      sink = client.sink
+      assert_kind_of(StatsD::Instrument::BatchedSink, sink)
+
+      underlying_sink = sink.instance_variable_get(:@sink)
+      connection = underlying_sink.connection
+      assert_kind_of(StatsD::Instrument::UdsConnection, connection)
+      assert_equal(65507, connection.instance_variable_get(:@max_packet_size))
+    ensure
+      server.close
+      File.unlink(socket_path) if File.exist?(socket_path)
+    end
+  end
 end

data/test/test_helper.rb

@@ -30,6 +30,15 @@ module StatsD
   end
 end
 
+# Add helper methods available to all tests
+module Minitest
+  class Test
+    def skip_on_jruby(message = "Test skipped on JRuby")
+      skip(message) if RUBY_ENGINE == "jruby"
+    end
+  end
+end
+
 StatsD.logger = Logger.new(File::NULL)
 
 Thread.abort_on_exception = true

data/test/udp_sink_test.rb

@@ -149,9 +149,27 @@ class UDPSinkTest < Minitest::Test
       UDPSocket.stubs(:new).returns(socket = mock("socket"))
 
       seq = sequence("connect_fail_connect_succeed")
+
+      # First attempt
+      socket.expects(:setsockopt)
+        .with(Socket::SOL_SOCKET, Socket::SO_SNDBUF, StatsD::Instrument::UdpConnection::DEFAULT_MAX_PACKET_SIZE)
+        .in_sequence(seq)
+      socket.expects(:getsockopt)
+        .with(Socket::SOL_SOCKET, Socket::SO_SNDBUF)
+        .returns(mock(int: StatsD::Instrument::UdpConnection::DEFAULT_MAX_PACKET_SIZE))
+        .in_sequence(seq)
       socket.expects(:connect).with("localhost", 8125).in_sequence(seq)
       socket.expects(:send).raises(Errno::EDESTADDRREQ).in_sequence(seq)
       socket.expects(:close).in_sequence(seq)
+
+      # Second attempt after error
+      socket.expects(:setsockopt)
+        .with(Socket::SOL_SOCKET, Socket::SO_SNDBUF, StatsD::Instrument::UdpConnection::DEFAULT_MAX_PACKET_SIZE)
+        .in_sequence(seq)
+      socket.expects(:getsockopt)
+        .with(Socket::SOL_SOCKET, Socket::SO_SNDBUF)
+        .returns(mock(int: StatsD::Instrument::UdpConnection::DEFAULT_MAX_PACKET_SIZE))
+        .in_sequence(seq)
       socket.expects(:connect).with("localhost", 8125).in_sequence(seq)
       socket.expects(:send).twice.returns(1).in_sequence(seq)
       socket.expects(:close).in_sequence(seq)
@@ -161,7 +179,8 @@ class UDPSinkTest < Minitest::Test
       udp_sink << "bar:1|c"
 
       assert_equal(
-        "[#{@sink_class}] Resetting connection because of " \
+        "[#{@sink_class}] [#{@sink_class.for_addr("localhost:8125").connection.class}] " \
+          "Resetting connection because of " \
           "Errno::EDESTADDRREQ: Destination address required\n",
         logs.string,
       )

data/test/uds_sink_test.rb

@@ -35,10 +35,6 @@ module UdsTestHelper
     @sink ||= build_sink(@socket_path)
   end
 
-  def skip_on_jruby(message = "JRuby does not support UNIX domain sockets")
-    skip(message) if RUBY_PLATFORM == "java"
-  end
-
   def read_datagrams(count, timeout: ENV["CI"] ? 5 : 1)
     datagrams = []
     count.times do

metadata

@@ -1,16 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: statsd-instrument
 version: !ruby/object:Gem::Version
-  version: 3.9.7
+  version: 3.9.8
 platform: ruby
+original_platform: ''
 authors:
 - Jesse Storimer
 - Tobias Lutke
 - Willem van Bergen
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-11-06 00:00:00.000000000 Z
+date: 2024-12-19 00:00:00.000000000 Z
 dependencies: []
 description: A StatsD client for Ruby apps. Provides metaprogramming methods to inject
   StatsD instrumentation into your code.
@@ -34,6 +34,7 @@ files:
 - CONTRIBUTING.md
 - Gemfile
 - LICENSE
+- Makefile
 - README.md
 - Rakefile
 - benchmark/README.md
@@ -49,6 +50,7 @@ files:
 - lib/statsd/instrument/batched_sink.rb
 - lib/statsd/instrument/capture_sink.rb
 - lib/statsd/instrument/client.rb
+- lib/statsd/instrument/connection_behavior.rb
 - lib/statsd/instrument/datagram.rb
 - lib/statsd/instrument/datagram_builder.rb
 - lib/statsd/instrument/dogstatsd_datagram.rb
@@ -115,7 +117,6 @@ licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -130,8 +131,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.23
-signing_key:
+rubygems_version: 3.6.1
 specification_version: 4
 summary: A StatsD client for Ruby apps
 test_files: