statsd-instrument 3.1.2 → 3.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 85b73b161dac9bc8839c3b820e9ea038d05078519fca692e971a48be7077e799
-  data.tar.gz: e2d5bfa763a5d53537494c8a5006327a93967d4d2b63f74a685094c551815c81
+  metadata.gz: 62f90038a90bccc54c0fe5ac2fe1a2449e8662183e9b9a0cac209e5ae22a07c7
+  data.tar.gz: e620363a10bff05710ce52f6869364e378a0979e2ef53cc86c56f8c7fe491d91
 SHA512:
-  metadata.gz: e6db90f921635692d1d19145e3b346029886022c679649fac3e6f5641411ddc557fcd2028baff7be4e0b47a23e53f97e4afe9af4ecb7e537dc0850b32352fbec
-  data.tar.gz: 6c8f12361a70596d49dc84b3feb6637bc0ff7e9d62f1e6e7876ec02111cc94a2b561947ac4bfda045f612f57951f9789e578447e67a0b8bda4beec4f0fc76576
+  metadata.gz: 18836250885562c7862db1e515c8f8433e43cf795b886800e8a40fae7e7ebead2a120656e6f7654d5c3b87c9f364d3861441593f42dfda7fa69479146b800842
+  data.tar.gz: 6915ee31b5bab72a8d52ef588f6fa90e1df4e821e1aaa96cf7523b1392d49bdc56003de4e3f09e818aee21e2d094e9671b2b2297e7b3f597ca193dafbf03824a

data/.github/workflows/benchmark.yml

@@ -5,18 +5,16 @@ on: push
 jobs:
   test:
     name: Send metric over UDP
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
 
     steps:
     - uses: actions/checkout@v1
 
-    - name: Setup Ruby
-      uses: actions/setup-ruby@v1
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
       with:
         ruby-version: 2.6
-
-    - name: Install dependencies
-      run: gem install bundler && bundle install --jobs 4 --retry 3
+        bundler-cache: true
 
     - name: Run benchmark on branch
       run: benchmark/send-metrics-to-local-udp-receiver

data/.github/workflows/cla.yml

@@ -0,0 +1,22 @@
+name: Contributor License Agreement (CLA)
+
+on:
+  pull_request_target:
+    types: [opened, synchronize]
+  issue_comment:
+    types: [created]
+
+jobs:
+  cla:
+    runs-on: ubuntu-latest
+    if: |
+      (github.event.issue.pull_request 
+        && !github.event.issue.pull_request.merged_at
+        && contains(github.event.comment.body, 'signed')
+      ) 
+      || (github.event.pull_request && !github.event.pull_request.merged)
+    steps:
+      - uses: Shopify/shopify-cla-action@v1
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          cla-token: ${{ secrets.CLA_TOKEN }}

data/.github/workflows/lint.yml

@@ -5,18 +5,16 @@ on: push
 jobs:
   test:
     name: Rubocop
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
 
     steps:
     - uses: actions/checkout@v1
 
-    - name: Setup Ruby
-      uses: actions/setup-ruby@v1
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
       with:
-        ruby-version: 2.7
-
-    - name: Install dependencies
-      run: gem install bundler && bundle install --jobs 4 --retry 3
+        ruby-version: 2.6
+        bundler-cache: true
 
     - name: Run Rubocop
       run: bin/rubocop

data/.github/workflows/tests.yml

@@ -4,12 +4,12 @@ on: push
 
 jobs:
   test:
-    name: Ruby ${{ matrix.ruby }} on ubuntu-18.04
-    runs-on: ubuntu-18.04
+    name: Ruby ${{ matrix.ruby }} on ubuntu-latest
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        ruby: ['2.6', '2.7', '3.0']
+        ruby: ['2.6', '2.7', '3.0', '3.1']
 
         # Windows on macOS builds started failing, so they are disabled for noew
         # platform: [windows-2019, macOS-10.14, ubuntu-18.04]
@@ -19,13 +19,11 @@ jobs:
     steps:
     - uses: actions/checkout@v1
 
-    - name: Setup Ruby
-      uses: actions/setup-ruby@v1
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby }}
-
-    - name: Install dependencies
-      run: gem install bundler && bundle install --jobs 4 --retry 3
+        bundler-cache: true
 
     - name: Run test suite
-      run: rake test
+      run: bundle exec rake test

data/.rubocop.yml

@@ -17,6 +17,12 @@ Naming/FileName:
   Exclude:
     - lib/statsd-instrument.rb
 
+Metrics/ParameterLists:
+  Enabled: false
+
+Style/WhileUntilModifier:
+  Enabled: false
+
 # Enable our own cops on our own repo
 
 StatsD/MetricReturnValue:

data/CHANGELOG.md

@@ -6,7 +6,23 @@ section below.
 
 ### Unreleased changes
 
-_Nothing yet_
+## Version 3.3.0
+
+- UDP Batching now has a max queue size and emitter threads will block if the queue
+  reaches the limit. This is to prevent the queue from growing unbounded.
+  More generally the UDP batching mode was optimized to improve throughput and to
+  flush the queue more eagerly (#309).
+- Added `STATSD_BUFFER_CAPACITY` configuration.
+- Added `STATSD_MAX_PACKET_SIZE` configuration.
+- Require `set` explicitly, to avoid breaking tests for users of this library (#311)
+
+## Version 3.2.1
+
+- Fix a bug in UDP batching that could cause the Ruby process to be stuck on exit (#291).
+
+## Version 3.2.0
+
+- Add `tag_error_class` option to `statsd_count_success` which tags the class of a thrown error
 
 ## Version 3.1.2
 

data/Gemfile

@@ -8,6 +8,6 @@ gem "minitest"
 gem "rspec"
 gem "mocha"
 gem "yard"
-gem "rubocop", ">= 1.0"
+gem "rubocop", [">= 1.0", "< 1.30"] # TODO: Our cops are broken by rubocop 1.30, we need to figure out why
 gem "rubocop-shopify", require: false
 gem "benchmark-ips"

data/README.md

@@ -27,7 +27,7 @@ The following environment variables are supported:
   explicitly, this will be determined based on other environment variables,
   like `RAILS_ENV` or `ENV`. The library will behave differently:
 
-  - In the **production** and **staging** environment, thre library will
+  - In the **production** and **staging** environment, the library will
     actually send UDP packets.
   - In the **test** environment, it will swallow all calls, but allows you to
     capture them for testing purposes. See below for notes on writing tests.
@@ -45,6 +45,14 @@ The following environment variables are supported:
 - `STATSD_FLUSH_INTERVAL`: (default: `1.0`) The interval in seconds at which
   events are sent in batch. Only applicable to the UDP configuration. If set
   to `0.0`, metrics are sent immediately.
+- `STATSD_BUFFER_CAPACITY`: (default: `5000`) The maximum amount of events that
+  may be buffered before emitting threads will start to block. Increasing this
+  value may help for application generating spikes of events. However if the
+  application emit events faster than they can be sent, increasing it won't help.
+- `STATSD_MAX_PACKET_SIZE`: (default: `1472`) The maximum size of UDP packets.
+  If your network is properly configured to handle larger packets you may try
+  to increase this value for better performance, but most network can't handle
+  larger packets.
 
 ## StatsD keys
 

data/benchmark/send-metrics-to-local-udp-receiver

@@ -16,28 +16,47 @@ else
 end
 
 intermediate_results_filename = "#{Dir.tmpdir}/statsd-instrument-benchmarks/#{File.basename($PROGRAM_NAME)}"
+log_filename = "#{Dir.tmpdir}/statsd-instrument-benchmarks/#{File.basename($PROGRAM_NAME)}.log"
 FileUtils.mkdir_p(File.dirname(intermediate_results_filename))
 
 # Set up an UDP listener to which we can send StatsD packets
 receiver = UDPSocket.new
 receiver.bind("localhost", 0)
 
-StatsD.singleton_client = StatsD::Instrument::Environment.new(
+log_file = File.open(log_filename, "w+", level: Logger::WARN)
+StatsD.logger = Logger.new(log_file)
+
+udp_client = StatsD::Instrument::Environment.new(ENV.to_h.merge(
   "STATSD_ADDR" => "#{receiver.addr[2]}:#{receiver.addr[1]}",
   "STATSD_IMPLEMENTATION" => "dogstatsd",
   "STATSD_ENV" => "production",
-).client
+  "STATSD_FLUSH_INTERVAL" => "0",
+)).client
+
+batched_udp_client = StatsD::Instrument::Environment.new(ENV.to_h.merge(
+  "STATSD_ADDR" => "#{receiver.addr[2]}:#{receiver.addr[1]}",
+  "STATSD_IMPLEMENTATION" => "dogstatsd",
+  "STATSD_ENV" => "production",
+)).client
+
+def send_metrics(client)
+  client.increment("StatsD.increment", 10)
+  client.measure("StatsD.measure") { 1 + 1 }
+  client.gauge("StatsD.gauge", 12.0, tags: ["foo:bar", "quc"])
+  client.set("StatsD.set", "value", tags: { foo: "bar", baz: "quc" })
+  if client.datagram_builder_class == StatsD::Instrument::DogStatsDDatagramBuilder
+    client.event("StasD.event", "12345")
+    client.service_check("StatsD.service_check", "ok")
+  end
+end
 
 report = Benchmark.ips do |bench|
-  bench.report("StatsD metrics to local UDP receiver (branch: #{branch}, sha: #{revision[0, 7]})") do
-    StatsD.increment("StatsD.increment", 10)
-    StatsD.measure("StatsD.measure") { 1 + 1 }
-    StatsD.gauge("StatsD.gauge", 12.0, tags: ["foo:bar", "quc"])
-    StatsD.set("StatsD.set", "value", tags: { foo: "bar", baz: "quc" })
-    if StatsD.singleton_client.datagram_builder_class == StatsD::Instrument::DogStatsDDatagramBuilder
-      StatsD.event("StasD.event", "12345")
-      StatsD.service_check("StatsD.service_check", "ok")
-    end
+  bench.report("local UDP sync (branch: #{branch}, sha: #{revision[0, 7]})") do
+    send_metrics(udp_client)
+  end
+
+  bench.report("local UDP batched (branch: #{branch}, sha: #{revision[0, 7]})") do
+    send_metrics(batched_udp_client)
   end
 
   # Store the results in between runs
@@ -57,3 +76,11 @@ elsif ENV["KEEP_RESULTS"]
 else
   File.unlink(intermediate_results_filename)
 end
+
+log_file.close
+logs = File.read(log_filename)
+unless logs.empty?
+  puts
+  puts "==== logs ===="
+  puts logs
+end

data/lib/statsd/instrument/assertions.rb

@@ -57,6 +57,7 @@ module StatsD
       def assert_no_statsd_calls(*metric_names, datagrams: nil, client: nil, &block)
         if datagrams.nil?
           raise LocalJumpError, "assert_no_statsd_calls requires a block" unless block_given?
+
           datagrams = capture_statsd_datagrams_with_exception_handling(client: client, &block)
         end
 
@@ -152,6 +153,7 @@ module StatsD
       def assert_statsd_expectations(expectations, datagrams: nil, client: nil, &block)
         if datagrams.nil?
           raise LocalJumpError, "assert_statsd_expectations requires a block" unless block_given?
+
           datagrams = capture_statsd_datagrams_with_exception_handling(client: client, &block)
         end
 

data/lib/statsd/instrument/batched_udp_sink.rb

@@ -6,11 +6,15 @@ module StatsD
     #   to become the new default in the next major release of this library.
     class BatchedUDPSink
       DEFAULT_FLUSH_INTERVAL = 1.0
-      MAX_PACKET_SIZE = 508
+      DEFAULT_THREAD_PRIORITY = 100
+      DEFAULT_FLUSH_THRESHOLD = 50
+      DEFAULT_BUFFER_CAPACITY = 5_000
+      # https://docs.datadoghq.com/developers/dogstatsd/high_throughput/?code-lang=ruby#ensure-proper-packet-sizes
+      DEFAULT_MAX_PACKET_SIZE = 1472
 
-      def self.for_addr(addr, flush_interval: DEFAULT_FLUSH_INTERVAL)
+      def self.for_addr(addr, **kwargs)
         host, port_as_string = addr.split(":", 2)
-        new(host, Integer(port_as_string), flush_interval: flush_interval)
+        new(host, Integer(port_as_string), **kwargs)
       end
 
       attr_reader :host, :port
@@ -21,10 +25,26 @@ module StatsD
         end
       end
 
-      def initialize(host, port, flush_interval: DEFAULT_FLUSH_INTERVAL)
+      def initialize(
+        host,
+        port,
+        flush_interval: DEFAULT_FLUSH_INTERVAL,
+        thread_priority: DEFAULT_THREAD_PRIORITY,
+        flush_threshold: DEFAULT_FLUSH_THRESHOLD,
+        buffer_capacity: DEFAULT_BUFFER_CAPACITY,
+        max_packet_size: DEFAULT_MAX_PACKET_SIZE
+      )
         @host = host
         @port = port
-        @dispatcher = Dispatcher.new(host, port, flush_interval)
+        @dispatcher = Dispatcher.new(
+          host,
+          port,
+          flush_interval,
+          flush_threshold,
+          buffer_capacity,
+          thread_priority,
+          max_packet_size,
+        )
         ObjectSpace.define_finalizer(self, self.class.finalize(@dispatcher))
       end
 
@@ -37,6 +57,10 @@ module StatsD
         self
       end
 
+      def shutdown(*args)
+        @dispatcher.shutdown(*args)
+      end
+
       class Dispatcher
         BUFFER_CLASS = if !::Object.const_defined?(:RUBY_ENGINE) || RUBY_ENGINE == "ruby"
           ::Array
@@ -50,24 +74,54 @@ module StatsD
           Concurrent::Array
         end
 
-        def initialize(host, port, flush_interval)
+        def initialize(host, port, flush_interval, flush_threshold, buffer_capacity, thread_priority, max_packet_size)
           @host = host
           @port = port
           @interrupted = false
           @flush_interval = flush_interval
+          @flush_threshold = flush_threshold
+          @buffer_capacity = buffer_capacity
+          @thread_priority = thread_priority
+          @max_packet_size = max_packet_size
           @buffer = BUFFER_CLASS.new
           @dispatcher_thread = Thread.new { dispatch }
+          @pid = Process.pid
+          @monitor = Monitor.new
+          @condition = @monitor.new_cond
         end
 
         def <<(datagram)
-          unless @dispatcher_thread&.alive?
-            # If the dispatcher thread is dead, we assume it is because
-            # the process was forked. So to avoid ending datagrams twice
-            # we clear the buffer.
-            @buffer.clear
-            @dispatcher_thread = Thread.new { dispatch }
+          if thread_healthcheck
+            @buffer << datagram
+
+            # To avoid sending too many signals when the thread is already flushing
+            # We only signal when the queue size is a multiple of `flush_threshold`
+            if @buffer.size % @flush_threshold == 0
+              wakeup_thread
+            end
+
+            # A SizedQueue would be perfect, except that it doesn't have a timeout
+            # Ref: https://bugs.ruby-lang.org/issues/18774
+            if @buffer.size >= @buffer_capacity
+              StatsD.logger.warn do
+                "[#{self.class.name}] Max buffer size reached (#{@buffer_capacity}), pausing " \
+                  "thread##{Thread.current.object_id}"
+              end
+              before = Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_millisecond)
+              @monitor.synchronize do
+                while @buffer.size >= @buffer_capacity && @dispatcher_thread.alive?
+                  @condition.wait(0.01)
+                end
+              end
+              duration = Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_millisecond) - before
+              StatsD.logger.warn do
+                "[#{self.class.name}] thread##{Thread.current.object_id} resumed after #{duration.round(2)}ms"
+              end
+            end
+          else
+            flush
           end
-          @buffer << datagram
+
           self
         end
 
@@ -82,6 +136,24 @@ module StatsD
 
         private
 
+        def wakeup_thread
+          begin
+            @monitor.synchronize do
+              @condition.signal
+            end
+          rescue ThreadError
+            # Can't synchronize from trap context
+            Thread.new { wakeup_thread }.join
+            return
+          end
+
+          begin
+            @dispatcher_thread&.run
+          rescue ThreadError # Somehow the thread just died
+            thread_healthcheck
+          end
+        end
+
         NEWLINE = "\n".b.freeze
         def flush
           return if @buffer.empty?
@@ -89,24 +161,54 @@ module StatsD
           datagrams = @buffer.shift(@buffer.size)
 
           until datagrams.empty?
-            packet = String.new(datagrams.pop, encoding: Encoding::BINARY, capacity: MAX_PACKET_SIZE)
+            packet = String.new(datagrams.shift, encoding: Encoding::BINARY, capacity: @max_packet_size)
 
-            until datagrams.empty? || packet.bytesize + datagrams.first.bytesize + 1 > MAX_PACKET_SIZE
+            until datagrams.empty? || packet.bytesize + datagrams.first.bytesize + 1 > @max_packet_size
               packet << NEWLINE << datagrams.shift
             end
-
             send_packet(packet)
           end
         end
 
+        def thread_healthcheck
+          # TODO: We have a race condition on JRuby / Truffle here. It could cause multiple
+          # dispatcher threads to be spawned, which would cause problems.
+          # However we can't simply lock here as we might be called from a trap context.
+          unless @dispatcher_thread&.alive?
+            # If the main the main thread is dead the VM is shutting down so we won't be able
+            # to spawn a new thread, so we fallback to sending our datagram directly.
+            return false unless Thread.main.alive?
+
+            # If the dispatcher thread is dead, it might be because the process was forked.
+            # So to avoid sending datagrams twice we clear the buffer.
+            if @pid != Process.pid
+              StatsD.logger.info { "[#{self.class.name}] Restarting the dispatcher thread after fork" }
+              @pid = Process.pid
+              @buffer.clear
+            else
+              StatsD.logger.info { "[#{self.class.name}] Restarting the dispatcher thread" }
+            end
+            @dispatcher_thread = Thread.new { dispatch }.tap { |t| t.priority = @thread_priority }
+          end
+          true
+        end
+
         def dispatch
           until @interrupted
             begin
               start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
               flush
+
+              # Other threads may have queued more events while we were doing IO
+              flush while @buffer.size > @flush_threshold
+
               next_sleep_duration = @flush_interval - (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start)
 
-              sleep(next_sleep_duration) if next_sleep_duration > 0
+              if next_sleep_duration > 0
+                @monitor.synchronize do
+                  @condition.wait(next_sleep_duration)
+                end
+              end
             rescue => error
               report_error(error)
             end
@@ -124,19 +226,21 @@ module StatsD
 
         def send_packet(packet)
           retried = false
-          socket.send(packet, 0)
-        rescue SocketError, IOError, SystemCallError => error
-          StatsD.logger.debug do
-            "[#{self.class.name}] Resetting connection because of #{error.class}: #{error.message}"
-          end
-          invalidate_socket
-          if retried
-            StatsD.logger.warning do
-              "[#{self.class.name}] Events were dropped because of #{error.class}: #{error.message}"
+          begin
+            socket.send(packet, 0)
+          rescue SocketError, IOError, SystemCallError => error
+            StatsD.logger.debug do
+              "[#{self.class.name}] Resetting connection because of #{error.class}: #{error.message}"
+            end
+            invalidate_socket
+            if retried
+              StatsD.logger.warn do
+                "[#{self.class.name}] Events were dropped because of #{error.class}: #{error.message}"
+              end
+            else
+              retried = true
+              retry
             end
-          else
-            retried = true
-            retry
           end
         end
 

data/lib/statsd/instrument/client.rb

@@ -200,6 +200,7 @@ module StatsD
       def increment(name, value = 1, sample_rate: nil, tags: nil, no_prefix: false)
         sample_rate ||= @default_sample_rate
         return StatsD::Instrument::VOID unless sample?(sample_rate)
+
         emit(datagram_builder(no_prefix: no_prefix).c(name, value, sample_rate, tags))
       end
 
@@ -217,6 +218,7 @@ module StatsD
 
         sample_rate ||= @default_sample_rate
         return StatsD::Instrument::VOID unless sample?(sample_rate)
+
         emit(datagram_builder(no_prefix: no_prefix).ms(name, value, sample_rate, tags))
       end
 
@@ -236,6 +238,7 @@ module StatsD
       def gauge(name, value, sample_rate: nil, tags: nil, no_prefix: false)
         sample_rate ||= @default_sample_rate
         return StatsD::Instrument::VOID unless sample?(sample_rate)
+
         emit(datagram_builder(no_prefix: no_prefix).g(name, value, sample_rate, tags))
       end
 
@@ -249,6 +252,7 @@ module StatsD
       def set(name, value, sample_rate: nil, tags: nil, no_prefix: false)
         sample_rate ||= @default_sample_rate
         return StatsD::Instrument::VOID unless sample?(sample_rate)
+
         emit(datagram_builder(no_prefix: no_prefix).s(name, value, sample_rate, tags))
       end
 
@@ -271,6 +275,7 @@ module StatsD
 
         sample_rate ||= @default_sample_rate
         return StatsD::Instrument::VOID unless sample?(sample_rate)
+
         emit(datagram_builder(no_prefix: no_prefix).d(name, value, sample_rate, tags))
       end
 
@@ -288,6 +293,7 @@ module StatsD
       def histogram(name, value, sample_rate: nil, tags: nil, no_prefix: false)
         sample_rate ||= @default_sample_rate
         return StatsD::Instrument::VOID unless sample?(sample_rate)
+
         emit(datagram_builder(no_prefix: no_prefix).h(name, value, sample_rate, tags))
       end
 

data/lib/statsd/instrument/datagram_builder.rb

@@ -67,10 +67,12 @@ module StatsD
       # @return [Array<String>, nil] the list of tags in canonical form.
       def normalize_tags(tags)
         return [] unless tags
+
         tags = tags.map { |k, v| "#{k}:#{v}" } if tags.is_a?(Hash)
 
         # Fast path when no string replacement is needed
         return tags unless tags.any? { |tag| /[|,]/.match?(tag) }
+
         tags.map { |tag| tag.tr("|,", "") }
       end
 
@@ -78,6 +80,7 @@ module StatsD
       def normalize_name(name)
         # Fast path when no normalization is needed to avoid copying the string
         return name unless /[:|@]/.match?(name)
+
         name.tr(":|@", "_")
       end
 

data/lib/statsd/instrument/environment.rb

@@ -79,7 +79,15 @@ module StatsD
       end
 
       def statsd_flush_interval
-        Float(env.fetch("STATSD_FLUSH_INTERVAL", 1.0))
+        Float(env.fetch("STATSD_FLUSH_INTERVAL", StatsD::Instrument::BatchedUDPSink::DEFAULT_FLUSH_INTERVAL))
+      end
+
+      def statsd_buffer_capacity
+        Float(env.fetch("STATSD_BUFFER_CAPACITY", StatsD::Instrument::BatchedUDPSink::DEFAULT_BUFFER_CAPACITY))
+      end
+
+      def statsd_max_packet_size
+        Float(env.fetch("STATSD_MAX_PACKET_SIZE", StatsD::Instrument::BatchedUDPSink::DEFAULT_MAX_PACKET_SIZE))
       end
 
       def client
@@ -90,7 +98,12 @@ module StatsD
         case environment
         when "production", "staging"
           if statsd_flush_interval > 0.0
-            StatsD::Instrument::BatchedUDPSink.for_addr(statsd_addr, flush_interval: statsd_flush_interval)
+            StatsD::Instrument::BatchedUDPSink.for_addr(
+              statsd_addr,
+              flush_interval: statsd_flush_interval,
+              buffer_capacity: statsd_buffer_capacity,
+              max_packet_size: statsd_max_packet_size,
+            )
           else
             StatsD::Instrument::UDPSink.for_addr(statsd_addr)
           end

data/lib/statsd/instrument/expectation.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "set"
+
 module StatsD
   module Instrument
     # @private
@@ -93,10 +95,12 @@ module StatsD
       #   to ensure that this logic matches the logic of the active datagram builder.
       def normalize_tags(tags)
         return [] unless tags
+
         tags = tags.map { |k, v| "#{k}:#{v}" } if tags.is_a?(Hash)
 
         # Fast path when no string replacement is needed
         return tags unless tags.any? { |tag| /[|,]/.match?(tag) }
+
         tags.map { |tag| tag.tr("|,", "") }
       end
     end

data/lib/statsd/instrument/helpers.rb

@@ -10,6 +10,22 @@ module StatsD
 
       # For backwards compatibility
       alias_method :capture_statsd_calls, :capture_statsd_datagrams
+
+      def self.add_tag(tags, key, value)
+        tags = tags.dup || {}
+
+        if tags.is_a?(String)
+          tags = tags.empty? ? "#{key}:#{value}" : "#{tags},#{key}:#{value}"
+        elsif tags.is_a?(Array)
+          tags << "#{key}:#{value}"
+        elsif tags.is_a?(Hash)
+          tags[key] = value
+        else
+          raise ArgumentError, "add_tag only supports string, array or hash, #{tags.class} provided"
+        end
+
+        tags
+      end
     end
   end
 end

data/lib/statsd/instrument/matchers.rb

@@ -49,7 +49,7 @@ module StatsD
             raise RSpec::Expectations::ExpectationNotMetError, "No StatsD calls for metric #{metric_name} were made."
           elsif options[:times] && options[:times] != metrics.length
             raise RSpec::Expectations::ExpectationNotMetError, "The numbers of StatsD calls for metric " \
-             "#{metric_name} was unexpected. Expected #{options[:times].inspect}, got #{metrics.length}"
+              "#{metric_name} was unexpected. Expected #{options[:times].inspect}, got #{metrics.length}"
           end
 
           [:sample_rate, :value, :tags].each do |expectation|

data/lib/statsd/instrument/rubocop/positional_arguments.rb

@@ -41,7 +41,7 @@ module RuboCop
         end
 
         def autocorrect(node)
-          -> (corrector) do
+          ->(corrector) do
             positional_arguments = if node.arguments.last.type == :block_pass
               node.arguments[2...node.arguments.length - 1]
             else

data/lib/statsd/instrument/rubocop.rb

@@ -39,6 +39,7 @@ module RuboCop
 
       def keyword_arguments(node)
         return nil if node.arguments.empty?
+
         last_argument = if node.arguments.last&.type == :block_pass
           node.arguments[node.arguments.length - 2]
         else

data/lib/statsd/instrument/statsd_datagram_builder.rb

@@ -11,6 +11,7 @@ module StatsD
 
       def normalize_tags(tags)
         raise NotImplementedError, "#{self.class.name} does not support tags" if tags
+
         super
       end
     end

data/lib/statsd/instrument/strict.rb

@@ -27,6 +27,7 @@ module StatsD
       def increment(key, value = 1, sample_rate: nil, tags: nil, no_prefix: false)
         raise ArgumentError, "StatsD.increment does not accept a block" if block_given?
         raise ArgumentError, "The value argument should be an integer, got #{value.inspect}" unless value.is_a?(Integer)
+
         check_tags_and_sample_rate(sample_rate, tags)
 
         super
@@ -35,6 +36,7 @@ module StatsD
       def gauge(key, value, sample_rate: nil, tags: nil, no_prefix: false)
         raise ArgumentError, "StatsD.increment does not accept a block" if block_given?
         raise ArgumentError, "The value argument should be an integer, got #{value.inspect}" unless value.is_a?(Numeric)
+
         check_tags_and_sample_rate(sample_rate, tags)
 
         super
@@ -43,6 +45,7 @@ module StatsD
       def histogram(key, value, sample_rate: nil, tags: nil, no_prefix: false)
         raise ArgumentError, "StatsD.increment does not accept a block" if block_given?
         raise ArgumentError, "The value argument should be an integer, got #{value.inspect}" unless value.is_a?(Numeric)
+
         check_tags_and_sample_rate(sample_rate, tags)
 
         super
@@ -50,6 +53,7 @@ module StatsD
 
       def set(key, value, sample_rate: nil, tags: nil, no_prefix: false)
         raise ArgumentError, "StatsD.set does not accept a block" if block_given?
+
         check_tags_and_sample_rate(sample_rate, tags)
 
         super

data/lib/statsd/instrument/udp_sink.rb

@@ -26,24 +26,27 @@ module StatsD
       def <<(datagram)
         with_socket { |socket| socket.send(datagram, 0) }
         self
-
-      rescue ThreadError
-        # In cases where a TERM or KILL signal has been sent, and we send stats as
-        # part of a signal handler, locks cannot be acquired, so we do our best
-        # to try and send the datagram without a lock.
-        socket.send(datagram, 0) > 0
-
       rescue SocketError, IOError, SystemCallError => error
         StatsD.logger.debug do
           "[StatsD::Instrument::UDPSink] Resetting connection because of #{error.class}: #{error.message}"
         end
         invalidate_socket
+        self
       end
 
       private
 
+      def synchronize(&block)
+        @mutex.synchronize(&block)
+      rescue ThreadError
+        # In cases where a TERM or KILL signal has been sent, and we send stats as
+        # part of a signal handler, locks cannot be acquired, so we do our best
+        # to try and send the datagram without a lock.
+        yield
+      end
+
       def with_socket
-        @mutex.synchronize { yield(socket) }
+        synchronize { yield(socket) }
       end
 
       def socket
@@ -55,7 +58,7 @@ module StatsD
       end
 
       def invalidate_socket
-        @mutex.synchronize do
+        synchronize do
           @socket = nil
         end
       end

data/lib/statsd/instrument/version.rb

@@ -2,6 +2,6 @@
 
 module StatsD
   module Instrument
-    VERSION = "3.1.2"
+    VERSION = "3.3.0"
   end
 end

data/lib/statsd/instrument.rb

@@ -104,24 +104,26 @@ module StatsD
     # @param method (see #statsd_measure)
     # @param name (see #statsd_measure)
     # @param metric_options (see #statsd_measure)
+    # @param tag_error_class add a <tt>error_class</tt> tag with the error class when an error is thrown
     # @yield You can pass a block to this method if you want to define yourself what is a successful call
     #   based on the return value of the method.
     # @yieldparam result The return value of the instrumented method.
     # @yieldreturn [Boolean] Return true iff the return value is considered a success, false otherwise.
     # @return [void]
     # @see #statsd_count_if
-    def statsd_count_success(method, name, sample_rate: nil, tags: nil, no_prefix: false, client: nil)
+    def statsd_count_success(method, name, sample_rate: nil,
+      tags: nil, no_prefix: false, client: nil, tag_error_class: false)
       add_to_method(method, name, :count_success) do
         define_method(method) do |*args, &block|
           truthiness = result = super(*args, &block)
-        rescue
+        rescue => error
           truthiness = false
           raise
         else
           if block_given?
             begin
               truthiness = yield(result)
-            rescue
+            rescue => error
               truthiness = false
             end
           end
@@ -130,6 +132,9 @@ module StatsD
           client ||= StatsD.singleton_client
           suffix = truthiness == false ? "failure" : "success"
           key = StatsD::Instrument.generate_metric_name(name, self, *args)
+
+          tags = Helpers.add_tag(tags, :error_class, error.class.name) if tag_error_class && error
+
           client.increment("#{key}.#{suffix}", sample_rate: sample_rate, tags: tags, no_prefix: no_prefix)
         end
       end

data/test/helpers_test.rb

@@ -35,8 +35,47 @@ class HelpersTest < Minitest::Test
     StatsD.gauge("gauge", 15)
 
     assert_equal(2, metrics.length)
-
   ensure
     StatsD.singleton_client = @old_client
   end
+
+  def test_add_tag_works_for_nil
+    assert_equal({ key: 123 }, StatsD::Instrument::Helpers.add_tag(nil, :key, 123))
+  end
+
+  def test_add_tag_works_for_hashes
+    assert_equal({ key: 123 }, StatsD::Instrument::Helpers.add_tag({}, :key, 123))
+
+    existing = { existing: 123 }
+    assert_equal({ existing: 123, new: 456 }, StatsD::Instrument::Helpers.add_tag(existing, :new, 456))
+
+    # ensure we do not modify the existing tags
+    assert_equal({ existing: 123 }, existing)
+  end
+
+  def test_add_tag_works_for_arrays
+    assert_equal(["key:123"], StatsD::Instrument::Helpers.add_tag([], :key, 123))
+
+    existing = ["existing:123"]
+    assert_equal(["existing:123", "new:456"], StatsD::Instrument::Helpers.add_tag(existing, :new, 456))
+
+    # ensure we do not modify the existing tags
+    assert_equal(["existing:123"], existing)
+  end
+
+  def test_add_tag_works_for_strings
+    assert_equal("key:123", StatsD::Instrument::Helpers.add_tag("", :key, 123))
+
+    existing = "existing:123"
+    assert_equal("existing:123,new:456", StatsD::Instrument::Helpers.add_tag(existing, :new, 456))
+
+    # ensure we do not modify the existing tags
+    assert_equal("existing:123", existing)
+  end
+
+  def test_add_tags_raises_for_other
+    assert_raises(ArgumentError, "add_tag only supports string, array or hash, Integer provided") do
+      StatsD::Instrument::Helpers.add_tag(1, :key, 123)
+    end
+  end
 end

data/test/statsd_instrumentation_test.rb

@@ -159,6 +159,38 @@ class StatsDInstrumentationTest < Minitest::Test
     ActiveMerchant::UniqueGateway.statsd_remove_count_success(:ssl_post, "ActiveMerchant.Gateway")
   end
 
+  def test_statsd_count_success_tag_error_class
+    ActiveMerchant::Base.statsd_count_success(:ssl_post, "ActiveMerchant.Base", tag_error_class: true)
+
+    assert_statsd_increment("ActiveMerchant.Base.success", tags: nil) do
+      ActiveMerchant::Base.new.ssl_post(true)
+    end
+
+    assert_statsd_increment("ActiveMerchant.Base.failure", tags: ["error_class:RuntimeError"]) do
+      assert_raises(RuntimeError, "Not OK") do
+        ActiveMerchant::Base.new.ssl_post(false)
+      end
+    end
+  ensure
+    ActiveMerchant::Base.statsd_remove_count_success(:ssl_post, "ActiveMerchant.Base")
+  end
+
+  def test_statsd_count_success_tag_error_class_is_opt_in
+    ActiveMerchant::Base.statsd_count_success(:ssl_post, "ActiveMerchant.Base")
+
+    assert_statsd_increment("ActiveMerchant.Base.success", tags: nil) do
+      ActiveMerchant::Base.new.ssl_post(true)
+    end
+
+    assert_statsd_increment("ActiveMerchant.Base.failure", tags: nil) do
+      assert_raises(RuntimeError, "Not OK") do
+        ActiveMerchant::Base.new.ssl_post(false)
+      end
+    end
+  ensure
+    ActiveMerchant::Base.statsd_remove_count_success(:ssl_post, "ActiveMerchant.Base")
+  end
+
   def test_statsd_count
     ActiveMerchant::Gateway.statsd_count(:ssl_post, "ActiveMerchant.Gateway.ssl_post")
 

data/test/udp_sink_test.rb

@@ -32,17 +32,19 @@ module UDPSinkTests
     refute(udp_sink.sample?(0.5))
   end
 
-  def test_parallelism
+  def test_concurrency
     udp_sink = build_sink(@host, @port)
-    50.times.map { |i| Thread.new { udp_sink << "foo:#{i}|c" << "bar:#{i}|c" } }
+    threads = 50.times.map { |i| Thread.new { udp_sink << "foo:#{i}|c" << "bar:#{i}|c" } }
     datagrams = []
 
     while @receiver.wait_readable(2)
-      datagram, _source = @receiver.recvfrom(4000)
+      datagram, _source = @receiver.recvfrom(4096)
       datagrams += datagram.split("\n")
     end
 
     assert_equal(100, datagrams.size)
+  ensure
+    threads&.each(&:kill)
   end
 
   class SimpleFormatter < ::Logger::Formatter
@@ -53,31 +55,39 @@ module UDPSinkTests
 
   def test_sends_datagram_in_signal_handler
     udp_sink = build_sink(@host, @port)
-    Signal.trap("USR1") { udp_sink << "exiting:1|c" }
-
-    pid = fork do
-      sleep(5)
+    Signal.trap("USR1") do
+      udp_sink << "exiting:1|c"
+      udp_sink << "exiting:1|d"
     end
 
+    Process.kill("USR1", Process.pid)
+    assert_equal(["exiting:1|c", "exiting:1|d"], read_datagrams(2))
+  ensure
     Signal.trap("USR1", "DEFAULT")
-
-    Process.kill("USR1", pid)
-    @receiver.wait_readable(1)
-    assert_equal("exiting:1|c", @receiver.recvfrom_nonblock(100).first)
-    Process.kill("KILL", pid)
-  rescue NotImplementedError
-    pass("Fork is not implemented on #{RUBY_PLATFORM}")
   end
 
   def test_sends_datagram_before_exit
     udp_sink = build_sink(@host, @port)
-    fork do
+    pid = fork do
       udp_sink << "exiting:1|c"
-      Process.exit(0)
+      udp_sink << "exiting:1|d"
     end
+    Process.wait(pid)
+    assert_equal(["exiting:1|c", "exiting:1|d"], read_datagrams(2))
+  rescue NotImplementedError
+    pass("Fork is not implemented on #{RUBY_PLATFORM}")
+  end
 
-    @receiver.wait_readable(1)
-    assert_equal("exiting:1|c", @receiver.recvfrom_nonblock(100).first)
+  def test_sends_datagram_in_at_exit_callback
+    udp_sink = build_sink(@host, @port)
+    pid = fork do
+      at_exit do
+        udp_sink << "exiting:1|c"
+        udp_sink << "exiting:1|d"
+      end
+    end
+    Process.wait(pid)
+    assert_equal(["exiting:1|c", "exiting:1|d"], read_datagrams(2))
   rescue NotImplementedError
     pass("Fork is not implemented on #{RUBY_PLATFORM}")
   end
@@ -86,11 +96,11 @@ module UDPSinkTests
     udp_sink = build_sink(@host, @port)
     fork do
       udp_sink << "exiting:1|c"
+      udp_sink << "exiting:1|d"
       Process.kill("TERM", Process.pid)
     end
 
-    @receiver.wait_readable(1)
-    assert_equal("exiting:1|c", @receiver.recvfrom_nonblock(100).first)
+    assert_equal(["exiting:1|c", "exiting:1|d"], read_datagrams(2))
   rescue NotImplementedError
     pass("Fork is not implemented on #{RUBY_PLATFORM}")
   end
@@ -101,6 +111,18 @@ module UDPSinkTests
     @sink_class.new(host, port)
   end
 
+  def read_datagrams(count, timeout: 2)
+    datagrams = []
+    count.times do
+      if @receiver.wait_readable(timeout)
+        datagrams += @receiver.recvfrom_nonblock(1000).first.lines(chomp: true)
+      else
+        break
+      end
+    end
+    datagrams
+  end
+
   class UDPSinkTest < Minitest::Test
     include UDPSinkTests
 
@@ -136,7 +158,7 @@ module UDPSinkTests
 
         assert_equal(
           "[#{@sink_class}] Resetting connection because of " \
-          "Errno::EDESTADDRREQ: Destination address required\n",
+            "Errno::EDESTADDRREQ: Destination address required\n",
           logs.string,
         )
       ensure
@@ -145,7 +167,7 @@ module UDPSinkTests
     end
   end
 
-  class BatchedUDPSinkTest < Minitest::Test
+  module BatchedUDPSinkTests
     include UDPSinkTests
 
     def setup
@@ -154,28 +176,63 @@ module UDPSinkTests
       @host = @receiver.addr[2]
       @port = @receiver.addr[1]
       @sink_class = StatsD::Instrument::BatchedUDPSink
+      @sinks = []
     end
 
     def teardown
       @receiver.close
+      @sinks.each(&:shutdown)
     end
 
-    def test_parallelism_buffering
+    private
+
+    def build_sink(host = @host, port = @port)
+      sink = @sink_class.new(host, port, flush_threshold: default_flush_threshold, buffer_capacity: 50)
+      @sinks << sink
+      sink
+    end
+
+    def default_flush_threshold
+      StatsD::Instrument::BatchedUDPSink::DEFAULT_FLUSH_THRESHOLD
+    end
+  end
+
+  class BatchedUDPSinkTest < Minitest::Test
+    include BatchedUDPSinkTests
+
+    def test_concurrency_buffering
       udp_sink = build_sink(@host, @port)
-      50.times.map do |i|
+      threads = 50.times.map do |i|
         Thread.new do
           udp_sink << "foo:#{i}|c" << "bar:#{i}|c" << "baz:#{i}|c" << "plop:#{i}|c"
         end
       end
+      threads.each(&:join)
+      assert_equal(200, read_datagrams(10, timeout: 2).size)
+    ensure
+      threads&.each(&:kill)
+    end
+  end
 
-      datagrams = []
+  class LowThresholdBatchedUDPSinkTest < Minitest::Test
+    include BatchedUDPSinkTests
 
-      while @receiver.wait_readable(2)
-        datagram, _source = @receiver.recvfrom(1000)
-        datagrams += datagram.split("\n")
-      end
+    def test_sends_datagram_when_termed
+      # When the main thread exit, the dispatcher thread is aborted
+      # and there's no exceptions or anything like that to rescue.
+      # So if the dispatcher thread poped some events from the buffer
+      # but didn't sent them yet, then they may be lost.
+      skip("Unfortunately this can't be guaranteed")
+    end
+    alias_method :test_sends_datagram_in_at_exit_callback, :test_sends_datagram_when_termed
+    alias_method :test_sends_datagram_before_exit, :test_sends_datagram_when_termed
+
+    private
 
-      assert_equal(200, datagrams.size)
+    # We run the same tests again, but this time we wake up the dispatcher
+    # thread on every call to make sure trap context is properly handled
+    def default_flush_threshold
+      1
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: statsd-instrument
 version: !ruby/object:Gem::Version
-  version: 3.1.2
+  version: 3.3.0
 platform: ruby
 authors:
 - Jesse Storimer
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-09-02 00:00:00.000000000 Z
+date: 2022-07-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -35,8 +35,8 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".github/CODEOWNERS"
-- ".github/probots.yml"
 - ".github/workflows/benchmark.yml"
+- ".github/workflows/cla.yml"
 - ".github/workflows/lint.yml"
 - ".github/workflows/tests.yml"
 - ".gitignore"
@@ -134,7 +134,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.20
+rubygems_version: 3.3.3
 signing_key: 
 specification_version: 4
 summary: A StatsD client for Ruby apps

data/.github/probots.yml

@@ -1,2 +0,0 @@
-enabled:
-  - cla