statsd-instrument 3.0.1 → 3.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3a26da2deac1a048fae156e2e8804ec5a423fde1e383ed1b3c446d34b0c553ec
-  data.tar.gz: edae81c0c408a10c4729d019930596e2a586abc17863b57642e3f2a0ff1b3084
+  metadata.gz: 85b73b161dac9bc8839c3b820e9ea038d05078519fca692e971a48be7077e799
+  data.tar.gz: e2d5bfa763a5d53537494c8a5006327a93967d4d2b63f74a685094c551815c81
 SHA512:
-  metadata.gz: 87037561be37e4fbf2cdee8e32564b35ae66b9d4c5a308e4784764d879171644ce2e0e5470fca8ac949b3e4ac3bff633e4bf95e83c37449dbbbe365543305bba
-  data.tar.gz: 9679d5832ffb8406ab800580bbab62ac3ba3d2b936ac9e841a755a90a9f09883ab62777f084a1ad6c979dbb4372f283cfed61e7e200dd32ee164da2673b82669
+  metadata.gz: e6db90f921635692d1d19145e3b346029886022c679649fac3e6f5641411ddc557fcd2028baff7be4e0b47a23e53f97e4afe9af4ecb7e537dc0850b32352fbec
+  data.tar.gz: 6c8f12361a70596d49dc84b3feb6637bc0ff7e9d62f1e6e7876ec02111cc94a2b561947ac4bfda045f612f57951f9789e578447e67a0b8bda4beec4f0fc76576

data/.github/workflows/lint.yml

@@ -0,0 +1,22 @@
+name: Lint
+
+on: push
+
+jobs:
+  test:
+    name: Rubocop
+    runs-on: ubuntu-18.04
+
+    steps:
+    - uses: actions/checkout@v1
+
+    - name: Setup Ruby
+      uses: actions/setup-ruby@v1
+      with:
+        ruby-version: 2.7
+
+    - name: Install dependencies
+      run: gem install bundler && bundle install --jobs 4 --retry 3
+
+    - name: Run Rubocop
+      run: bin/rubocop

data/.github/workflows/{ci.yml → tests.yml}

@@ -9,27 +9,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # Windows on macOS builds started failing, so they are disabled for noew
+        ruby: ['2.6', '2.7', '3.0']
 
-        ruby: [2.4, 2.5, 2.6, 2.7]
+        # Windows on macOS builds started failing, so they are disabled for noew
         # platform: [windows-2019, macOS-10.14, ubuntu-18.04]
-
         # exclude:
-        # The Windows environment does not support older Ruby versions. We only test against the latest version
-        # - platform: windows-2019
-        #   ruby: 2.3
-        # - platform: windows-2019
-        #   ruby: 2.4
-        # - platform: windows-2019
-        #   ruby: 2.5
-
-        # On macOS, we only test against the Ruby version macOS ships with (2.3)
-        # - platform: macOS-10.14
-        #   ruby: 2.4
-        # - platform: macOS-10.14
-        #   ruby: 2.5
-        # - platform: macOS-10.14
-        #   ruby: 2.6
+        # ...
 
     steps:
     - uses: actions/checkout@v1
@@ -44,6 +29,3 @@ jobs:
 
     - name: Run test suite
       run: rake test
-
-    - name: Run Rubocop
-      run: bin/rubocop

data/.rubocop.yml

@@ -5,8 +5,9 @@ require:
   - ./lib/statsd/instrument/rubocop.rb
 
 AllCops:
-  TargetRubyVersion: 2.4
+  TargetRubyVersion: 2.7
   UseCache: true
+  SuggestExtensions: false
   CacheRootDirectory: tmp/rubocop
   Exclude:
     - statsd-instrument.gemspec

data/CHANGELOG.md

@@ -8,6 +8,24 @@ section below.
 
 _Nothing yet_
 
+## Version 3.1.2
+
+ - Fix bug when passing custom client to expectation.
+
+## Version 3.1.1
+
+ - Improved flushing of buffered datagrams on process exit when using UDP batching.
+
+## Version 3.1.0
+
+- Introduced UDP batching using a dispatcher thread, and made it the
+  production default.
+- Dropped support for Ruby 2.4 and 2.5.
+
+## Version 3.0.2
+
+- Properly handle no_prefix when using StatsD assertions.
+
 ## Version 3.0.1
 
 - Fix metaprograming methods to not print keyword argument warnings on

data/Gemfile

@@ -3,13 +3,11 @@
 source "https://rubygems.org"
 gemspec
 
-gem 'rake'
-gem 'minitest'
-gem 'rspec'
-gem 'mocha'
-gem 'yard'
-gem 'rubocop'
-gem 'rubocop-shopify', require: false
-
-# benchmark-ips save! method is not part of a released version yet.
-gem 'benchmark-ips', git: 'https://github.com/evanphx/benchmark-ips', branch: 'master'
+gem "rake"
+gem "minitest"
+gem "rspec"
+gem "mocha"
+gem "yard"
+gem "rubocop", ">= 1.0"
+gem "rubocop-shopify", require: false
+gem "benchmark-ips"

data/README.md

@@ -1,6 +1,6 @@
 # StatsD client for Ruby apps
 
-This is a ruby client for statsd (http://github.com/etsy/statsd). It provides
+This is a ruby client for statsd (https://github.com/statsd/statsd). It provides
 a lightweight way to track and measure metrics in your application.
 
 We call out to statsd by sending data over a UDP socket. UDP sockets are fast,
@@ -10,8 +10,8 @@ because it means your code doesn't get bogged down trying to log statistics.
 We send data to statsd several times per request and haven't noticed a
 performance hit.
 
-For more information about StatsD, see the [README of the Etsy
-project](http://github.com/etsy/statsd).
+For more information about StatsD, see the [README of the StatsD
+project](https://github.com/statsd/statsd).
 
 ## Configuration
 
@@ -20,7 +20,7 @@ The following environment variables are supported:
 
 - `STATSD_ADDR`: (default `localhost:8125`) The address to send the StatsD UDP
   datagrams to.
-- `STATSD_IMPLEMENTATION`: (default: `statsd`). The StatsD implementation you
+- `STATSD_IMPLEMENTATION`: (default: `datadog`). The StatsD implementation you
   are using. `statsd`, `statsite` and `datadog` are supported. Some features
   are only available on certain implementations,
 - `STATSD_ENV`: The environment StatsD will run in. If this is not set
@@ -42,6 +42,9 @@ The following environment variables are supported:
   overridden in a metric method call.
 - `STATSD_DEFAULT_TAGS`: A comma-separated list of tags to apply to all metrics.
   (Note: tags are not supported by all implementations.)
+- `STATSD_FLUSH_INTERVAL`: (default: `1.0`) The interval in seconds at which
+  events are sent in batch. Only applicable to the UDP configuration. If set
+  to `0.0`, metrics are sent immediately.
 
 ## StatsD keys
 

data/Rakefile

@@ -1,12 +1,12 @@
 # frozen_string_literal: true
 
-require 'bundler/gem_tasks'
-require 'rake/testtask'
+require "bundler/gem_tasks"
+require "rake/testtask"
 
-Rake::TestTask.new('test') do |t|
-  t.ruby_opts << '-r rubygems'
-  t.libs << 'lib' << 'test'
-  t.test_files = FileList['test/**/*_test.rb']
+Rake::TestTask.new("test") do |t|
+  t.ruby_opts << "-r rubygems"
+  t.libs << "lib" << "test"
+  t.test_files = FileList["test/**/*_test.rb"]
 end
 
 task(default: :test)

data/benchmark/send-metrics-to-dev-null-log

@@ -1,14 +1,14 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-require 'bundler/setup'
-require 'tmpdir'
-require 'benchmark/ips'
+require "bundler/setup"
+require "tmpdir"
+require "benchmark/ips"
 
 revision = %x(git rev-parse HEAD).rstrip
-master_revision = %x(git rev-parse origin/master).rstrip
-branch = if revision == master_revision
-  'master'
+base_revision = %x(git rev-parse origin/master).rstrip
+branch = if revision == base_revision
+  "master"
 else
   %x(git rev-parse --abbrev-ref HEAD).rstrip
 end
@@ -16,18 +16,18 @@ end
 intermediate_results_filename = "#{Dir.tmpdir}/statsd-instrument-benchmarks/#{File.basename($PROGRAM_NAME)}"
 FileUtils.mkdir_p(File.dirname(intermediate_results_filename))
 
-ENV['ENV'] = "development"
-require 'statsd-instrument'
+ENV["ENV"] = "development"
+require "statsd-instrument"
 StatsD.logger = Logger.new(File::NULL)
 
 report = Benchmark.ips do |bench|
   bench.report("StatsD metrics to /dev/null log (branch: #{branch}, sha: #{revision[0, 7]})") do
-    StatsD.increment('StatsD.increment', 10, sample_rate: 15)
-    StatsD.measure('StatsD.measure') { 1 + 1 }
-    StatsD.gauge('StatsD.gauge', 12.0, tags: ["foo:bar", "quc"])
-    StatsD.set('StatsD.set', 'value', tags: { foo: 'bar', baz: 'quc' })
+    StatsD.increment("StatsD.increment", 10, sample_rate: 15)
+    StatsD.measure("StatsD.measure") { 1 + 1 }
+    StatsD.gauge("StatsD.gauge", 12.0, tags: ["foo:bar", "quc"])
+    StatsD.set("StatsD.set", "value", tags: { foo: "bar", baz: "quc" })
     if StatsD.singleton_client.datagram_builder_class == StatsD::Instrument::DogStatsDDatagramBuilder
-      StatsD.event('StasD.event', "12345")
+      StatsD.event("StasD.event", "12345")
       StatsD.service_check("StatsD.service_check", "ok")
     end
   end
@@ -41,7 +41,7 @@ if report.entries.length == 1
   puts
   puts "To compare the performance of this revision against another revision (e.g. master),"
   puts "check out a different branch and run this benchmark script again."
-elsif ENV['KEEP_RESULTS']
+elsif ENV["KEEP_RESULTS"]
   puts
   puts "The intermediate results have been stored in #{intermediate_results_filename}"
 else

data/benchmark/send-metrics-to-local-udp-receiver

@@ -1,16 +1,16 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-require 'bundler/setup'
-require 'benchmark/ips'
-require 'tmpdir'
-require 'socket'
-require 'statsd-instrument'
+require "bundler/setup"
+require "benchmark/ips"
+require "tmpdir"
+require "socket"
+require "statsd-instrument"
 
 revision = %x(git rev-parse HEAD).rstrip
-master_revision = %x(git rev-parse origin/master).rstrip
-branch = if revision == master_revision
-  'master'
+base_revision = %x(git rev-parse origin/master).rstrip
+branch = if revision == base_revision
+  "master"
 else
   %x(git rev-parse --abbrev-ref HEAD).rstrip
 end
@@ -20,22 +20,22 @@ FileUtils.mkdir_p(File.dirname(intermediate_results_filename))
 
 # Set up an UDP listener to which we can send StatsD packets
 receiver = UDPSocket.new
-receiver.bind('localhost', 0)
+receiver.bind("localhost", 0)
 
 StatsD.singleton_client = StatsD::Instrument::Environment.new(
-  'STATSD_ADDR' => "#{receiver.addr[2]}:#{receiver.addr[1]}",
-  'STATSD_IMPLEMENTATION' => 'dogstatsd',
-  'STATSD_ENV' => 'production',
+  "STATSD_ADDR" => "#{receiver.addr[2]}:#{receiver.addr[1]}",
+  "STATSD_IMPLEMENTATION" => "dogstatsd",
+  "STATSD_ENV" => "production",
 ).client
 
 report = Benchmark.ips do |bench|
   bench.report("StatsD metrics to local UDP receiver (branch: #{branch}, sha: #{revision[0, 7]})") do
-    StatsD.increment('StatsD.increment', 10)
-    StatsD.measure('StatsD.measure') { 1 + 1 }
-    StatsD.gauge('StatsD.gauge', 12.0, tags: ["foo:bar", "quc"])
-    StatsD.set('StatsD.set', 'value', tags: { foo: 'bar', baz: 'quc' })
+    StatsD.increment("StatsD.increment", 10)
+    StatsD.measure("StatsD.measure") { 1 + 1 }
+    StatsD.gauge("StatsD.gauge", 12.0, tags: ["foo:bar", "quc"])
+    StatsD.set("StatsD.set", "value", tags: { foo: "bar", baz: "quc" })
     if StatsD.singleton_client.datagram_builder_class == StatsD::Instrument::DogStatsDDatagramBuilder
-      StatsD.event('StasD.event', "12345")
+      StatsD.event("StasD.event", "12345")
       StatsD.service_check("StatsD.service_check", "ok")
     end
   end
@@ -51,7 +51,7 @@ if report.entries.length == 1
   puts
   puts "To compare the performance of this revision against another revision (e.g. master),"
   puts "check out a different branch and run this benchmark script again."
-elsif ENV['KEEP_RESULTS']
+elsif ENV["KEEP_RESULTS"]
   puts
   puts "The intermediate results have been stored in #{intermediate_results_filename}"
 else

data/lib/statsd/instrument/assertions.rb

@@ -61,7 +61,7 @@ module StatsD
         end
 
         datagrams.select! { |metric| metric_names.include?(metric.name) } unless metric_names.empty?
-        assert(datagrams.empty?, "No StatsD calls for metric #{datagrams.map(&:name).join(', ')} expected.")
+        assert(datagrams.empty?, "No StatsD calls for metric #{datagrams.map(&:name).join(", ")} expected.")
       end
 
       # Asserts that a given counter metric occurred inside the provided block.
@@ -74,7 +74,7 @@ module StatsD
       # @raise [Minitest::Assertion] If an exception occurs, or if the metric did
       #   not occur as specified during the execution the block.
       def assert_statsd_increment(metric_name, value = nil, datagrams: nil, client: nil, **options, &block)
-        expectation = StatsD::Instrument::Expectation.increment(metric_name, value, **options)
+        expectation = StatsD::Instrument::Expectation.increment(metric_name, value, client: client, **options)
         assert_statsd_expectation(expectation, datagrams: datagrams, client: client, &block)
       end
 
@@ -86,7 +86,7 @@ module StatsD
       # @return [void]
       # @raise (see #assert_statsd_increment)
       def assert_statsd_measure(metric_name, value = nil, datagrams: nil, client: nil, **options, &block)
-        expectation = StatsD::Instrument::Expectation.measure(metric_name, value, **options)
+        expectation = StatsD::Instrument::Expectation.measure(metric_name, value, client: client, **options)
         assert_statsd_expectation(expectation, datagrams: datagrams, client: client, &block)
       end
 
@@ -98,7 +98,7 @@ module StatsD
       # @return [void]
       # @raise (see #assert_statsd_increment)
       def assert_statsd_gauge(metric_name, value = nil, datagrams: nil, client: nil, **options, &block)
-        expectation = StatsD::Instrument::Expectation.gauge(metric_name, value, **options)
+        expectation = StatsD::Instrument::Expectation.gauge(metric_name, value, client: client, **options)
         assert_statsd_expectation(expectation, datagrams: datagrams, client: client, &block)
       end
 
@@ -110,7 +110,7 @@ module StatsD
       # @return [void]
       # @raise (see #assert_statsd_increment)
       def assert_statsd_histogram(metric_name, value = nil, datagrams: nil, client: nil, **options, &block)
-        expectation = StatsD::Instrument::Expectation.histogram(metric_name, value, **options)
+        expectation = StatsD::Instrument::Expectation.histogram(metric_name, value, client: client, **options)
         assert_statsd_expectation(expectation, datagrams: datagrams, client: client, &block)
       end
 
@@ -122,7 +122,7 @@ module StatsD
       # @return [void]
       # @raise (see #assert_statsd_increment)
       def assert_statsd_distribution(metric_name, value = nil, datagrams: nil, client: nil, **options, &block)
-        expectation = StatsD::Instrument::Expectation.distribution(metric_name, value, **options)
+        expectation = StatsD::Instrument::Expectation.distribution(metric_name, value, client: client, **options)
         assert_statsd_expectation(expectation, datagrams: datagrams, client: client, &block)
       end
 
@@ -134,7 +134,7 @@ module StatsD
       # @return [void]
       # @raise (see #assert_statsd_increment)
       def assert_statsd_set(metric_name, value = nil, datagrams: nil, client: nil, **options, &block)
-        expectation = StatsD::Instrument::Expectation.set(metric_name, value, **options)
+        expectation = StatsD::Instrument::Expectation.set(metric_name, value, client: client, **options)
         assert_statsd_expectation(expectation, datagrams: datagrams, client: client, &block)
       end
 

data/lib/statsd/instrument/batched_udp_sink.rb

@@ -0,0 +1,159 @@
+# frozen_string_literal: true
+
+module StatsD
+  module Instrument
+    # @note This class is part of the new Client implementation that is intended
+    #   to become the new default in the next major release of this library.
+    class BatchedUDPSink
+      DEFAULT_FLUSH_INTERVAL = 1.0
+      MAX_PACKET_SIZE = 508
+
+      def self.for_addr(addr, flush_interval: DEFAULT_FLUSH_INTERVAL)
+        host, port_as_string = addr.split(":", 2)
+        new(host, Integer(port_as_string), flush_interval: flush_interval)
+      end
+
+      attr_reader :host, :port
+
+      class << self
+        def finalize(dispatcher)
+          proc { dispatcher.shutdown }
+        end
+      end
+
+      def initialize(host, port, flush_interval: DEFAULT_FLUSH_INTERVAL)
+        @host = host
+        @port = port
+        @dispatcher = Dispatcher.new(host, port, flush_interval)
+        ObjectSpace.define_finalizer(self, self.class.finalize(@dispatcher))
+      end
+
+      def sample?(sample_rate)
+        sample_rate == 1.0 || rand < sample_rate
+      end
+
+      def <<(datagram)
+        @dispatcher << datagram
+        self
+      end
+
+      class Dispatcher
+        BUFFER_CLASS = if !::Object.const_defined?(:RUBY_ENGINE) || RUBY_ENGINE == "ruby"
+          ::Array
+        else
+          begin
+            gem("concurrent-ruby")
+          rescue Gem::MissingSpecError
+            raise Gem::MissingSpecError, "statsd-instrument depends on `concurrent-ruby` on #{RUBY_ENGINE}"
+          end
+          require "concurrent/array"
+          Concurrent::Array
+        end
+
+        def initialize(host, port, flush_interval)
+          @host = host
+          @port = port
+          @interrupted = false
+          @flush_interval = flush_interval
+          @buffer = BUFFER_CLASS.new
+          @dispatcher_thread = Thread.new { dispatch }
+        end
+
+        def <<(datagram)
+          unless @dispatcher_thread&.alive?
+            # If the dispatcher thread is dead, we assume it is because
+            # the process was forked. So to avoid ending datagrams twice
+            # we clear the buffer.
+            @buffer.clear
+            @dispatcher_thread = Thread.new { dispatch }
+          end
+          @buffer << datagram
+          self
+        end
+
+        def shutdown(wait = @flush_interval * 2)
+          @interrupted = true
+          if @dispatcher_thread&.alive?
+            @dispatcher_thread.join(wait)
+          else
+            flush
+          end
+        end
+
+        private
+
+        NEWLINE = "\n".b.freeze
+        def flush
+          return if @buffer.empty?
+
+          datagrams = @buffer.shift(@buffer.size)
+
+          until datagrams.empty?
+            packet = String.new(datagrams.pop, encoding: Encoding::BINARY, capacity: MAX_PACKET_SIZE)
+
+            until datagrams.empty? || packet.bytesize + datagrams.first.bytesize + 1 > MAX_PACKET_SIZE
+              packet << NEWLINE << datagrams.shift
+            end
+
+            send_packet(packet)
+          end
+        end
+
+        def dispatch
+          until @interrupted
+            begin
+              start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+              flush
+              next_sleep_duration = @flush_interval - (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start)
+
+              sleep(next_sleep_duration) if next_sleep_duration > 0
+            rescue => error
+              report_error(error)
+            end
+          end
+
+          flush
+          invalidate_socket
+        end
+
+        def report_error(error)
+          StatsD.logger.error do
+            "[#{self.class.name}] The dispatcher thread encountered an error #{error.class}: #{error.message}"
+          end
+        end
+
+        def send_packet(packet)
+          retried = false
+          socket.send(packet, 0)
+        rescue SocketError, IOError, SystemCallError => error
+          StatsD.logger.debug do
+            "[#{self.class.name}] Resetting connection because of #{error.class}: #{error.message}"
+          end
+          invalidate_socket
+          if retried
+            StatsD.logger.warning do
+              "[#{self.class.name}] Events were dropped because of #{error.class}: #{error.message}"
+            end
+          else
+            retried = true
+            retry
+          end
+        end
+
+        def socket
+          @socket ||= begin
+            socket = UDPSocket.new
+            socket.connect(@host, @port)
+            socket
+          end
+        end
+
+        def invalidate_socket
+          @socket&.close
+        ensure
+          @socket = nil
+        end
+      end
+    end
+  end
+end