statsd-instrument 3.0.2 → 3.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 032db29a498269753044a778380369fc44d3f87b3143e051df69f1d8a480595a
-  data.tar.gz: 10d0ccc0cb4039d8846862e82283de004a7fb2b770648fb165281dd5e2d5f68c
+  metadata.gz: 3a481f6cd0897b93e3e992fc5315b496c033a98a8cccd02948146cb638e0a6b2
+  data.tar.gz: ab454f2fecc89aa662ac2f16632fe2fc3c4e4b2b102d3c4058642e5706bf64c1
 SHA512:
-  metadata.gz: 61b4c7cf6c84144767201dea7491cfa217dbd48b45821e6f1173eac8d45de6c9ddd9239ce50491839425d8cbe672f69c9ebfc9263916ec2b8a027fab43824f41
-  data.tar.gz: e726adb888e41bdfc8c9049c36b3df9b0f6ee2328a7a9e04f871652483dd57ade4ec04bd11e7010b275d77c733917c10441481c588c58bc9cb573b9364648dac
+  metadata.gz: f3fae39281925135491d66805ff8dada34b0ea798d3403fdf0e6d3b9c1e2d3eda4a78295d7eea046ba7a0db55fb5d357d72e853558f12fdf81c71eff3713c816
+  data.tar.gz: 852e0838027594b8451d4fb617ca9aa5f614b570c50180bd513fb2986419adf3e9ea4f59ce86e63fbe799ccc3a07720954c87778c9a9dca63cd62fb339906d43

data/.github/workflows/lint.yml

@@ -0,0 +1,22 @@
+name: Lint
+
+on: push
+
+jobs:
+  test:
+    name: Rubocop
+    runs-on: ubuntu-18.04
+
+    steps:
+    - uses: actions/checkout@v1
+
+    - name: Setup Ruby
+      uses: actions/setup-ruby@v1
+      with:
+        ruby-version: 2.6
+
+    - name: Install dependencies
+      run: gem install bundler && bundle install --jobs 4 --retry 3
+
+    - name: Run Rubocop
+      run: bin/rubocop

data/.github/workflows/{ci.yml → tests.yml}

@@ -9,27 +9,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # Windows on macOS builds started failing, so they are disabled for noew
+        ruby: ['2.6', '2.7', '3.0']
 
-        ruby: [2.4, 2.5, 2.6, 2.7]
+        # Windows on macOS builds started failing, so they are disabled for noew
         # platform: [windows-2019, macOS-10.14, ubuntu-18.04]
-
         # exclude:
-        # The Windows environment does not support older Ruby versions. We only test against the latest version
-        # - platform: windows-2019
-        #   ruby: 2.3
-        # - platform: windows-2019
-        #   ruby: 2.4
-        # - platform: windows-2019
-        #   ruby: 2.5
-
-        # On macOS, we only test against the Ruby version macOS ships with (2.3)
-        # - platform: macOS-10.14
-        #   ruby: 2.4
-        # - platform: macOS-10.14
-        #   ruby: 2.5
-        # - platform: macOS-10.14
-        #   ruby: 2.6
+        # ...
 
     steps:
     - uses: actions/checkout@v1
@@ -44,6 +29,3 @@ jobs:
 
     - name: Run test suite
       run: rake test
-
-    - name: Run Rubocop
-      run: bin/rubocop

data/.rubocop.yml

@@ -5,7 +5,7 @@ require:
   - ./lib/statsd/instrument/rubocop.rb
 
 AllCops:
-  TargetRubyVersion: 2.4
+  TargetRubyVersion: 2.6
   UseCache: true
   CacheRootDirectory: tmp/rubocop
   Exclude:

data/CHANGELOG.md

@@ -8,6 +8,12 @@ section below.
 
 _Nothing yet_
 
+## Version 3.1.0
+
+- Introduced UDP batching using a dispatcher thread, and made it the
+  production default.
+- Dropped support for Ruby 2.4 and 2.5.
+
 ## Version 3.0.2
 
 - Properly handle no_prefix when using StatsD assertions.

data/Gemfile

@@ -3,13 +3,11 @@
 source "https://rubygems.org"
 gemspec
 
-gem 'rake'
-gem 'minitest'
-gem 'rspec'
-gem 'mocha'
-gem 'yard'
-gem 'rubocop', '>= 1.0'
-gem 'rubocop-shopify', require: false
-
-# benchmark-ips save! method is not part of a released version yet.
-gem 'benchmark-ips', git: 'https://github.com/evanphx/benchmark-ips', branch: 'master'
+gem "rake"
+gem "minitest"
+gem "rspec"
+gem "mocha"
+gem "yard"
+gem "rubocop", ">= 1.0"
+gem "rubocop-shopify", require: false
+gem "benchmark-ips"

data/README.md

@@ -42,6 +42,9 @@ The following environment variables are supported:
   overridden in a metric method call.
 - `STATSD_DEFAULT_TAGS`: A comma-separated list of tags to apply to all metrics.
   (Note: tags are not supported by all implementations.)
+- `STATSD_FLUSH_INTERVAL`: (default: `1.0`) The interval in seconds at which
+  events are sent in batch. Only applicable to the UDP configuration. If set
+  to `0.0`, metrics are sent immediately.
 
 ## StatsD keys
 

data/Rakefile

@@ -1,12 +1,12 @@
 # frozen_string_literal: true
 
-require 'bundler/gem_tasks'
-require 'rake/testtask'
+require "bundler/gem_tasks"
+require "rake/testtask"
 
-Rake::TestTask.new('test') do |t|
-  t.ruby_opts << '-r rubygems'
-  t.libs << 'lib' << 'test'
-  t.test_files = FileList['test/**/*_test.rb']
+Rake::TestTask.new("test") do |t|
+  t.ruby_opts << "-r rubygems"
+  t.libs << "lib" << "test"
+  t.test_files = FileList["test/**/*_test.rb"]
 end
 
 task(default: :test)

data/benchmark/send-metrics-to-dev-null-log

@@ -1,14 +1,14 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-require 'bundler/setup'
-require 'tmpdir'
-require 'benchmark/ips'
+require "bundler/setup"
+require "tmpdir"
+require "benchmark/ips"
 
 revision = %x(git rev-parse HEAD).rstrip
 master_revision = %x(git rev-parse origin/master).rstrip
 branch = if revision == master_revision
-  'master'
+  "master"
 else
   %x(git rev-parse --abbrev-ref HEAD).rstrip
 end
@@ -16,18 +16,18 @@ end
 intermediate_results_filename = "#{Dir.tmpdir}/statsd-instrument-benchmarks/#{File.basename($PROGRAM_NAME)}"
 FileUtils.mkdir_p(File.dirname(intermediate_results_filename))
 
-ENV['ENV'] = "development"
-require 'statsd-instrument'
+ENV["ENV"] = "development"
+require "statsd-instrument"
 StatsD.logger = Logger.new(File::NULL)
 
 report = Benchmark.ips do |bench|
   bench.report("StatsD metrics to /dev/null log (branch: #{branch}, sha: #{revision[0, 7]})") do
-    StatsD.increment('StatsD.increment', 10, sample_rate: 15)
-    StatsD.measure('StatsD.measure') { 1 + 1 }
-    StatsD.gauge('StatsD.gauge', 12.0, tags: ["foo:bar", "quc"])
-    StatsD.set('StatsD.set', 'value', tags: { foo: 'bar', baz: 'quc' })
+    StatsD.increment("StatsD.increment", 10, sample_rate: 15)
+    StatsD.measure("StatsD.measure") { 1 + 1 }
+    StatsD.gauge("StatsD.gauge", 12.0, tags: ["foo:bar", "quc"])
+    StatsD.set("StatsD.set", "value", tags: { foo: "bar", baz: "quc" })
     if StatsD.singleton_client.datagram_builder_class == StatsD::Instrument::DogStatsDDatagramBuilder
-      StatsD.event('StasD.event', "12345")
+      StatsD.event("StasD.event", "12345")
       StatsD.service_check("StatsD.service_check", "ok")
     end
   end
@@ -41,7 +41,7 @@ if report.entries.length == 1
   puts
   puts "To compare the performance of this revision against another revision (e.g. master),"
   puts "check out a different branch and run this benchmark script again."
-elsif ENV['KEEP_RESULTS']
+elsif ENV["KEEP_RESULTS"]
   puts
   puts "The intermediate results have been stored in #{intermediate_results_filename}"
 else

data/benchmark/send-metrics-to-local-udp-receiver

@@ -1,16 +1,16 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-require 'bundler/setup'
-require 'benchmark/ips'
-require 'tmpdir'
-require 'socket'
-require 'statsd-instrument'
+require "bundler/setup"
+require "benchmark/ips"
+require "tmpdir"
+require "socket"
+require "statsd-instrument"
 
 revision = %x(git rev-parse HEAD).rstrip
 master_revision = %x(git rev-parse origin/master).rstrip
 branch = if revision == master_revision
-  'master'
+  "master"
 else
   %x(git rev-parse --abbrev-ref HEAD).rstrip
 end
@@ -20,22 +20,22 @@ FileUtils.mkdir_p(File.dirname(intermediate_results_filename))
 
 # Set up an UDP listener to which we can send StatsD packets
 receiver = UDPSocket.new
-receiver.bind('localhost', 0)
+receiver.bind("localhost", 0)
 
 StatsD.singleton_client = StatsD::Instrument::Environment.new(
-  'STATSD_ADDR' => "#{receiver.addr[2]}:#{receiver.addr[1]}",
-  'STATSD_IMPLEMENTATION' => 'dogstatsd',
-  'STATSD_ENV' => 'production',
+  "STATSD_ADDR" => "#{receiver.addr[2]}:#{receiver.addr[1]}",
+  "STATSD_IMPLEMENTATION" => "dogstatsd",
+  "STATSD_ENV" => "production",
 ).client
 
 report = Benchmark.ips do |bench|
   bench.report("StatsD metrics to local UDP receiver (branch: #{branch}, sha: #{revision[0, 7]})") do
-    StatsD.increment('StatsD.increment', 10)
-    StatsD.measure('StatsD.measure') { 1 + 1 }
-    StatsD.gauge('StatsD.gauge', 12.0, tags: ["foo:bar", "quc"])
-    StatsD.set('StatsD.set', 'value', tags: { foo: 'bar', baz: 'quc' })
+    StatsD.increment("StatsD.increment", 10)
+    StatsD.measure("StatsD.measure") { 1 + 1 }
+    StatsD.gauge("StatsD.gauge", 12.0, tags: ["foo:bar", "quc"])
+    StatsD.set("StatsD.set", "value", tags: { foo: "bar", baz: "quc" })
     if StatsD.singleton_client.datagram_builder_class == StatsD::Instrument::DogStatsDDatagramBuilder
-      StatsD.event('StasD.event', "12345")
+      StatsD.event("StasD.event", "12345")
       StatsD.service_check("StatsD.service_check", "ok")
     end
   end
@@ -51,7 +51,7 @@ if report.entries.length == 1
   puts
   puts "To compare the performance of this revision against another revision (e.g. master),"
   puts "check out a different branch and run this benchmark script again."
-elsif ENV['KEEP_RESULTS']
+elsif ENV["KEEP_RESULTS"]
   puts
   puts "The intermediate results have been stored in #{intermediate_results_filename}"
 else

data/lib/statsd-instrument.rb

@@ -1,3 +1,3 @@
 # frozen_string_literal: true
 
-require 'statsd/instrument'
+require "statsd/instrument"

data/lib/statsd/instrument.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-require 'socket'
-require 'logger'
-require 'forwardable'
+require "socket"
+require "logger"
+require "forwardable"
 
 # The `StatsD` module contains low-level metrics for collecting metrics and
 # sending them to the backend.
@@ -31,7 +31,7 @@ module StatsD
     # @private
     # @return [String]
     def self.generate_metric_name(name, callee, *args)
-      name.respond_to?(:call) ? name.call(callee, args).gsub('::', '.') : name.gsub('::', '.')
+      name.respond_to?(:call) ? name.call(callee, args).gsub("::", ".") : name.gsub("::", ".")
     end
 
     # Even though this method is considered private, and is no longer used internally,
@@ -113,26 +113,24 @@ module StatsD
     def statsd_count_success(method, name, sample_rate: nil, tags: nil, no_prefix: false, client: nil)
       add_to_method(method, name, :count_success) do
         define_method(method) do |*args, &block|
-          begin
-            truthiness = result = super(*args, &block)
-          rescue
-            truthiness = false
-            raise
-          else
-            if block_given?
-              begin
-                truthiness = yield(result)
-              rescue
-                truthiness = false
-              end
+          truthiness = result = super(*args, &block)
+        rescue
+          truthiness = false
+          raise
+        else
+          if block_given?
+            begin
+              truthiness = yield(result)
+            rescue
+              truthiness = false
             end
-            result
-          ensure
-            client ||= StatsD.singleton_client
-            suffix = truthiness == false ? 'failure' : 'success'
-            key = StatsD::Instrument.generate_metric_name(name, self, *args)
-            client.increment("#{key}.#{suffix}", sample_rate: sample_rate, tags: tags, no_prefix: no_prefix)
           end
+          result
+        ensure
+          client ||= StatsD.singleton_client
+          suffix = truthiness == false ? "failure" : "success"
+          key = StatsD::Instrument.generate_metric_name(name, self, *args)
+          client.increment("#{key}.#{suffix}", sample_rate: sample_rate, tags: tags, no_prefix: no_prefix)
         end
       end
     end
@@ -152,27 +150,25 @@ module StatsD
     def statsd_count_if(method, name, sample_rate: nil, tags: nil, no_prefix: false, client: nil)
       add_to_method(method, name, :count_if) do
         define_method(method) do |*args, &block|
-          begin
-            truthiness = result = super(*args, &block)
-          rescue
-            truthiness = false
-            raise
-          else
-            if block_given?
-              begin
-                truthiness = yield(result)
-              rescue
-                truthiness = false
-              end
-            end
-            result
-          ensure
-            if truthiness
-              client ||= StatsD.singleton_client
-              key = StatsD::Instrument.generate_metric_name(name, self, *args)
-              client.increment(key, sample_rate: sample_rate, tags: tags, no_prefix: no_prefix)
+          truthiness = result = super(*args, &block)
+        rescue
+          truthiness = false
+          raise
+        else
+          if block_given?
+            begin
+              truthiness = yield(result)
+            rescue
+              truthiness = false
             end
           end
+          result
+        ensure
+          if truthiness
+            client ||= StatsD.singleton_client
+            key = StatsD::Instrument.generate_metric_name(name, self, *args)
+            client.increment(key, sample_rate: sample_rate, tags: tags, no_prefix: no_prefix)
+          end
         end
       end
     end
@@ -347,21 +343,22 @@ module StatsD
   end
 end
 
-require 'statsd/instrument/version'
-require 'statsd/instrument/client'
-require 'statsd/instrument/datagram'
-require 'statsd/instrument/dogstatsd_datagram'
-require 'statsd/instrument/datagram_builder'
-require 'statsd/instrument/statsd_datagram_builder'
-require 'statsd/instrument/dogstatsd_datagram_builder'
-require 'statsd/instrument/null_sink'
-require 'statsd/instrument/udp_sink'
-require 'statsd/instrument/capture_sink'
-require 'statsd/instrument/log_sink'
-require 'statsd/instrument/environment'
-require 'statsd/instrument/helpers'
-require 'statsd/instrument/assertions'
-require 'statsd/instrument/expectation'
-require 'statsd/instrument/matchers' if defined?(::RSpec)
-require 'statsd/instrument/railtie' if defined?(::Rails::Railtie)
-require 'statsd/instrument/strict' if ENV['STATSD_STRICT_MODE']
+require "statsd/instrument/version"
+require "statsd/instrument/client"
+require "statsd/instrument/datagram"
+require "statsd/instrument/dogstatsd_datagram"
+require "statsd/instrument/datagram_builder"
+require "statsd/instrument/statsd_datagram_builder"
+require "statsd/instrument/dogstatsd_datagram_builder"
+require "statsd/instrument/null_sink"
+require "statsd/instrument/udp_sink"
+require "statsd/instrument/batched_udp_sink"
+require "statsd/instrument/capture_sink"
+require "statsd/instrument/log_sink"
+require "statsd/instrument/environment"
+require "statsd/instrument/helpers"
+require "statsd/instrument/assertions"
+require "statsd/instrument/expectation"
+require "statsd/instrument/matchers" if defined?(::RSpec)
+require "statsd/instrument/railtie" if defined?(::Rails::Railtie)
+require "statsd/instrument/strict" if ENV["STATSD_STRICT_MODE"]

data/lib/statsd/instrument/assertions.rb

@@ -61,7 +61,7 @@ module StatsD
         end
 
         datagrams.select! { |metric| metric_names.include?(metric.name) } unless metric_names.empty?
-        assert(datagrams.empty?, "No StatsD calls for metric #{datagrams.map(&:name).join(', ')} expected.")
+        assert(datagrams.empty?, "No StatsD calls for metric #{datagrams.map(&:name).join(", ")} expected.")
       end
 
       # Asserts that a given counter metric occurred inside the provided block.

data/lib/statsd/instrument/batched_udp_sink.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+module StatsD
+  module Instrument
+    # @note This class is part of the new Client implementation that is intended
+    #   to become the new default in the next major release of this library.
+    class BatchedUDPSink
+      DEFAULT_FLUSH_INTERVAL = 1.0
+      MAX_PACKET_SIZE = 508
+
+      def self.for_addr(addr, flush_interval: DEFAULT_FLUSH_INTERVAL)
+        host, port_as_string = addr.split(":", 2)
+        new(host, Integer(port_as_string), flush_interval: flush_interval)
+      end
+
+      attr_reader :host, :port
+
+      class << self
+        def finalize(dispatcher)
+          proc { dispatcher.shutdown }
+        end
+      end
+
+      def initialize(host, port, flush_interval: DEFAULT_FLUSH_INTERVAL)
+        @host = host
+        @port = port
+        @dispatcher = Dispatcher.new(host, port, flush_interval)
+        ObjectSpace.define_finalizer(self, self.class.finalize(@dispatcher))
+      end
+
+      def sample?(sample_rate)
+        sample_rate == 1 || rand < sample_rate
+      end
+
+      def <<(datagram)
+        @dispatcher << datagram
+        self
+      end
+
+      class Dispatcher
+        BUFFER_CLASS = if !::Object.const_defined?(:RUBY_ENGINE) || RUBY_ENGINE == "ruby"
+          ::Array
+        else
+          begin
+            gem("concurrent-ruby")
+          rescue Gem::MissingSpecError
+            raise Gem::MissingSpecError, "statsd-instrument depends on `concurrent-ruby` on #{RUBY_ENGINE}"
+          end
+          require "concurrent/array"
+          Concurrent::Array
+        end
+
+        def initialize(host, port, flush_interval)
+          @host = host
+          @port = port
+          @interrupted = false
+          @flush_interval = flush_interval
+          @buffer = BUFFER_CLASS.new
+          @dispatcher_thread = Thread.new { dispatch }
+        end
+
+        def <<(datagram)
+          unless @dispatcher_thread&.alive?
+            # If the dispatcher thread is dead, we assume it is because
+            # the process was forked. So to avoid ending datagrams twice
+            # we clear the buffer.
+            @buffer.clear
+            @dispatcher_thread = Thread.new { dispatch }
+          end
+          @buffer << datagram
+          self
+        end
+
+        def shutdown
+          @interrupted = true
+        end
+
+        private
+
+        NEWLINE = "\n".b.freeze
+        def flush
+          return if @buffer.empty?
+
+          datagrams = @buffer.shift(@buffer.size)
+
+          until datagrams.empty?
+            packet = String.new(datagrams.pop, encoding: Encoding::BINARY, capacity: MAX_PACKET_SIZE)
+
+            until datagrams.empty? || packet.bytesize + datagrams.first.bytesize + 1 > MAX_PACKET_SIZE
+              packet << NEWLINE << datagrams.shift
+            end
+
+            send_packet(packet)
+          end
+        end
+
+        def dispatch
+          until @interrupted
+            begin
+              start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+              flush
+              next_sleep_duration = @flush_interval - (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start)
+
+              sleep(next_sleep_duration) if next_sleep_duration > 0
+            rescue => error
+              report_error(error)
+            end
+          end
+
+          flush
+          invalidate_socket
+        end
+
+        def report_error(error)
+          StatsD.logger.error do
+            "[#{self.class.name}] The dispatcher thread encountered an error #{error.class}: #{error.message}"
+          end
+        end
+
+        def send_packet(packet)
+          retried = false
+          socket.send(packet, 0)
+        rescue SocketError, IOError, SystemCallError => error
+          StatsD.logger.debug do
+            "[#{self.class.name}] Resetting connection because of #{error.class}: #{error.message}"
+          end
+          invalidate_socket
+          if retried
+            StatsD.logger.warning do
+              "[#{self.class.name}] Events were dropped because of #{error.class}: #{error.message}"
+            end
+          else
+            retried = true
+            retry
+          end
+        end
+
+        def socket
+          @socket ||= begin
+            socket = UDPSocket.new
+            socket.connect(@host, @port)
+            socket
+          end
+        end
+
+        def invalidate_socket
+          @socket&.close
+        ensure
+          @socket = nil
+        end
+      end
+    end
+  end
+end