statsd-instrument 3.11.0 → 3.11.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1f2b45a3e809ef0e5ca26b7faf52512632705f62120b3b73987a10282e19ed76
-  data.tar.gz: 7d7574e77adea4dc54fcf0355b83e1392623bcf2d4ecccd60a2cc101f62c84bb
+  metadata.gz: 51853f9d4b70fc39649e251e7e93ff3d77b2bcca1d776182f82de8219ea86886
+  data.tar.gz: fce22a816682f0bc25510b6a68248a64c93ba983e7640b57ec01dbb0b985c9d2
 SHA512:
-  metadata.gz: bb416ef206605fb8204acec2b0501e4722a2bd51d81fbc1083c4a4a96203a2838dd329355908ee83b64b7a3943f8f8af862cbf7d8433363d4452e68f14292f95
-  data.tar.gz: bf0597922ce2dbe8837ef29b0de1f7c4fa1ca0e9623dc4d15628661b49059577d31499c46164a14e98744c5e765d68b407a15f14153c212ec245f3d90ed691e7
+  metadata.gz: 4a06bd60427001260e206243f4db74ecb3d8225c1c5e3f8f9764239bc40901457cf621d7f5fbd0936d39fa0765145499672a9fb0a9c22025ac8611f7051f7154
+  data.tar.gz: f74279e49141844c564d896e1d04b88b6b9f66585e56401a4ee52f7f4cb8dcc5fbe6a99b8427fbfa45ca1be843fef5a8c9357fb629129372880ac5020aceab31

data/.github/workflows/tests.yml

@@ -9,7 +9,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: ['2.7', '3.0', '3.1', '3.2', '3.3', 'ruby-head', 'jruby-9.4.9.0', 'truffleruby-25.0.0']
+        ruby: ['3.0', '3.1', '3.2', '3.3', '3.4', 'ruby-head', 'jruby-10.1.1.0', 'truffleruby-34.0.1']
 
     steps:
     - uses: actions/checkout@v4

data/CHANGELOG.md

@@ -6,6 +6,16 @@ section below.
 
 ## Unreleased changes
 
+## Version 3.11.2
+
+- Widen the `CompiledMetric` tag-combination cache key from 32 to 57 bits, eliminating
+  recurring hash collisions on hot metrics while keeping all intermediates within Fixnum
+  range (no new allocations on the emit path).
+
+## Version 3.11.1
+
+- Fix `CompiledMetric::Distribution` applying `sample_rate` twice.
+
 ## Version 3.11.0
 
 - [#418](https://github.com/Shopify/statsd-instrument/pull/418) - Prevent misuse of `CompiledMetric` definitions by requiring a subclass when defining one.

data/Gemfile

@@ -13,15 +13,12 @@ gem "rubocop-shopify", require: false
 gem "benchmark-ips"
 gem "dogstatsd-ruby", "~> 5.0", require: false
 gem "simplecov", require: false, group: :test
+# logger is no longer bundled with newer Ruby and JRuby releases.
+gem "logger"
 
 platform :mri do
   # only if Ruby is MRI && >= 3.2
   if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("3.2")
     gem "vernier", require: false
   end
-
-  # From Ruby >= 3.5, logger is not part of the stdlib anymore
-  if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("3.5")
-    gem "logger"
-  end
 end

data/lib/statsd/instrument/client.rb

@@ -285,6 +285,25 @@ module StatsD
         StatsD::Instrument::VOID
       end
 
+      # Emits a pre-sampled compiled distribution metric.
+      # This method is used when the generated compiled metric method already made
+      # the sampling decision before measuring block latency.
+      #
+      # @param precompiled_datagram [StatsD::Instrument::CompiledMetric::PrecompiledDatagram]
+      #   The precompiled metric datagram
+      # @param value [Numeric] The metric value
+      # @return [void]
+      # @api private
+      def emit_presampled_precompiled_distribution_metric(precompiled_datagram, value)
+        if @enable_aggregation
+          @aggregator.aggregate_precompiled_timing_metric(precompiled_datagram, value)
+          return StatsD::Instrument::VOID
+        end
+
+        emit(precompiled_datagram.to_datagram(value))
+        StatsD::Instrument::VOID
+      end
+
       # Emits a precompiled metric for high-performance use cases.
       # This method is used by {StatsD::Instrument::CompiledMetric::Gauge} to emit metrics
       # with minimal allocations.

data/lib/statsd/instrument/compiled_metric.rb

@@ -177,6 +177,10 @@ module StatsD
           default_val = default_value
           default_val_assignment = default_val.nil? ? "" : " = #{default_val.inspect}"
           allow_block = allow_measuring_latency
+          # Block-enabled compiled metrics run through generate_block_handler first,
+          # which applies sample_rate before emitting. Use the presampled client helper
+          # here so distributions don't make a second sampling decision.
+          client_emit_method = allow_block ? "emit_presampled_precompiled_#{method}_metric" : "emit_precompiled_#{method}_metric"
 
           method_code = <<~RUBY
             def self.#{method}(__value__#{default_val_assignment}, #{tag_names.map { |name| "#{name}:" }.join(", ")})
@@ -185,10 +189,13 @@ module StatsD
 
               # Compute hash of tag values for cache lookup using rotate-left + XOR.
               # Rotation makes it order-dependent (unlike plain XOR), preventing collisions
-              # when tag values are swapped. We mask to 32 bits to avoid Bignum allocations
-              # from left shifts on 64-bit hash values.
-              __cache_key__ = #{tag_names.first}.hash & 0xFFFFFFFF
-              #{tag_names.drop(1).map { |name| "__cache_key__ = (((__cache_key__ << 5) | (__cache_key__ >> 27)) ^ #{name}.hash) & 0xFFFFFFFF" }.join("\n")}
+              # when tag values are swapped. The mask keeps 57 bits of entropy, wide enough
+              # that birthday collisions are negligible at real tag-combination volumes.
+              # 57 bits is the widest key for which the intermediate (__cache_key__ << 5)
+              # stays within Fixnum range (2**62 - 1 on 64-bit CRuby) and never allocates
+              # a Bignum.
+              __cache_key__ = #{tag_names.first}.hash & 0x01FFFFFFFFFFFFFF
+              #{tag_names.drop(1).map { |name| "__cache_key__ = (((__cache_key__ << 5) | (__cache_key__ >> 52)) ^ #{name}.hash) & 0x01FFFFFFFFFFFFFF" }.join("\n")}
 
               # Look up or create a PrecompiledDatagram
               __datagram__ =
@@ -218,7 +225,7 @@ module StatsD
 
               __datagram__ ||= PrecompiledDatagram.new([#{tag_names.join(", ")}], @datagram_blueprint, @sample_rate)
 
-              @singleton_client.emit_precompiled_#{method}_metric(__datagram__, __value__)
+              @singleton_client.#{client_emit_method}(__datagram__, __value__)
               __return_value__
             end
           RUBY
@@ -233,12 +240,16 @@ module StatsD
           method = method_name
           default_val = default_value
           allow_block = allow_measuring_latency
+          # Block-enabled compiled metrics run through generate_block_handler first,
+          # which applies sample_rate before emitting. Use the presampled client helper
+          # here so distributions don't make a second sampling decision.
+          client_emit_method = allow_block ? "emit_presampled_precompiled_#{method}_metric" : "emit_precompiled_#{method}_metric"
 
           instance_eval(<<~RUBY, __FILE__, __LINE__ + 1)
             def self.#{method}(__value__ = #{default_val.inspect})
               __return_value__ = StatsD::Instrument::VOID
               #{generate_block_handler if allow_block}
-              @singleton_client.emit_precompiled_#{method}_metric(@static_datagram, __value__)
+              @singleton_client.#{client_emit_method}(@static_datagram, __value__)
               __return_value__
             end
           RUBY

data/lib/statsd/instrument/datagram.rb

@@ -77,7 +77,7 @@ module StatsD
         (?:\|\#(?<tags>(?:[^\|,]+(?:,[^\|,]+)*)))?
         \n? # In some implementations, the datagram may include a trailing newline.
         \z
-      }x
+      }x.freeze
 
       def parsed_datagram
         @parsed ||= if (match_info = PARSER.match(@source))

data/lib/statsd/instrument/dogstatsd_datagram.rb

@@ -69,7 +69,7 @@ module StatsD
         (?:\|m:(?<message>[^\|]+))?
         \n? # In some implementations, the datagram may include a trailing newline.
         \z
-      }x
+      }x.freeze
 
       # |k:my-key|p:low|s:source|t:success|
       EVENT_PARSER = %r{
@@ -84,7 +84,7 @@ module StatsD
         (?:\|\#(?<tags>(?:[^\|,]+(?:,[^\|,]+)*)))?
         \n? # In some implementations, the datagram may include a trailing newline.
         \z
-      }x
+      }x.freeze
 
       PARSER = Regexp.union(StatsD::Instrument::Datagram::PARSER, SERVICE_CHECK_PARSER, EVENT_PARSER)
     end

data/lib/statsd/instrument/rubocop/metric_return_value.rb

@@ -19,7 +19,7 @@ module RuboCop
 
         MSG = "Do not use the return value of StatsD metric methods"
 
-        INVALID_PARENTS = [:lvasgn, :array, :pair, :send, :return, :yield]
+        INVALID_PARENTS = [:lvasgn, :array, :pair, :send, :return, :yield].freeze
 
         def on_send(node)
           if metric_method?(node) && node.arguments.last&.type != :block_pass

data/lib/statsd/instrument/rubocop.rb

@@ -3,7 +3,7 @@
 module RuboCop
   module Cop
     module StatsD
-      METRIC_METHODS = [:increment, :gauge, :measure, :set, :histogram, :distribution, :key_value]
+      METRIC_METHODS = [:increment, :gauge, :measure, :set, :histogram, :distribution, :key_value].freeze
 
       METAPROGRAMMING_METHODS = [
         :statsd_measure,
@@ -11,7 +11,7 @@ module RuboCop
         :statsd_count_success,
         :statsd_count_if,
         :statsd_count,
-      ]
+      ].freeze
 
       SINGLETON_CONFIGURATION_METHODS = [
         :backend,
@@ -22,7 +22,7 @@ module RuboCop
         :"default_tags=",
         :default_sample_rate,
         :"default_sample_rate=",
-      ]
+      ].freeze
 
       private
 

data/lib/statsd/instrument/version.rb

@@ -2,6 +2,6 @@
 
 module StatsD
   module Instrument
-    VERSION = "3.11.0"
+    VERSION = "3.11.2"
   end
 end

data/test/compiled_metric/distribution_test.rb

@@ -230,6 +230,43 @@ class CompiledMetricDistributionTest < Minitest::Test
     assert_equal(["env:production", "region:us-east", "service:web"], datagram.tags.sort)
   end
 
+  def test_sampled_distribution_value_uses_one_sampling_decision
+    sample_rate = 0.5
+    @sink.expects(:sample?).with(sample_rate).once.returns(true)
+
+    metric = Class.new(StatsD::Instrument::CompiledMetric::Distribution) do
+      define(
+        name: "foo.bar",
+        tags: { shop_id: Integer },
+        sample_rate: sample_rate,
+      )
+    end
+
+    metric.distribution(5, shop_id: 123)
+
+    assert_equal(1, @sink.datagrams.size)
+    assert_equal("test.foo.bar:5|d|@0.5|#shop_id:123", @sink.datagrams.first.source)
+  end
+
+  def test_sampled_distribution_block_uses_one_sampling_decision
+    sample_rate = 0.5
+    @sink.expects(:sample?).with(sample_rate).once.returns(true)
+    Process.stubs(:clock_gettime).with(Process::CLOCK_MONOTONIC, :float_millisecond).returns(100.0, 125.0)
+
+    metric = Class.new(StatsD::Instrument::CompiledMetric::Distribution) do
+      define(
+        name: "foo.bar",
+        sample_rate: sample_rate,
+      )
+    end
+
+    returned_value = metric.distribution { :result }
+
+    assert_equal(:result, returned_value)
+    assert_equal(1, @sink.datagrams.size)
+    assert_equal("test.foo.bar:25.0|d|@0.5", @sink.datagrams.first.source)
+  end
+
   def test_latency_as_value_when_block_provided
     metric = Class.new(StatsD::Instrument::CompiledMetric::Distribution) do
       define(
@@ -415,18 +452,17 @@ class CompiledMetricDistributionWithAggregationTest < Minitest::Test
   end
 
   def test_sample_rate_with_aggregation
-    # When aggregating with sample_rate, sampling happens before aggregation
-    # This test verifies that with a sample_rate >0, a subset of distributions are aggregated
+    sample_rate = 0.5
+    @sink.expects(:sample?).with(sample_rate).times(5).returns(false, true, false, false, true)
+
     metric = Class.new(StatsD::Instrument::CompiledMetric::Distribution) do
       define(
         name: "foo.bar",
         static_tags: { service: "web" },
-        sample_rate: 0.5,
+        sample_rate: sample_rate,
       )
     end
 
-    metric.stubs(:sample?).returns(false, true, false, false, true)
-
     metric.distribution(1)
     metric.distribution(2)
     metric.distribution(3)

data/test/compiled_metric_test.rb

@@ -285,10 +285,10 @@ class CompiledMetricDefinitionTest < Minitest::Test
 
     cache = metric.instance_variable_get(:@tag_combination_cache)
 
-    # Compute cache keys using the rotate-left + XOR formula (32-bit bounded)
-    # For single tag, it's the hash value masked to 32 bits
-    cache_key_for_1 = 1.hash & 0xFFFFFFFF
-    cache_key_for_2 = 2.hash & 0xFFFFFFFF
+    # Compute cache keys using the rotate-left + XOR formula (57-bit bounded)
+    # For single tag, it's the hash value masked to 57 bits
+    cache_key_for_1 = 1.hash & 0x01FFFFFFFFFFFFFF
+    cache_key_for_2 = 2.hash & 0x01FFFFFFFFFFFFFF
 
     # Store the cached datagram under the collision key
     cached_datagram = cache[cache_key_for_1]
@@ -312,6 +312,77 @@ class CompiledMetricDefinitionTest < Minitest::Test
     assert_includes(hash_collision_metric.tags, "metric_name:foo.bar")
   end
 
+  def test_cache_key_space_is_wider_than_32_bits
+    metric = Class.new(StatsD::Instrument::CompiledMetric::Counter) do
+      define(
+        name: "foo.bar",
+        tags: { shop_id: Integer },
+      )
+    end
+
+    # Hash seeds are process-local, so find a pair of tag values at runtime that
+    # shares its low 32 hash bits (collided under the previous 32-bit cache key)
+    # while still differing across the full 57-bit key space.
+    seen = {}
+    pair = nil
+    i = 0
+    until pair
+      key = i.hash & 0xFFFFFFFF
+      other = seen[key]
+      if other && other != i && (other.hash & 0x01FFFFFFFFFFFFFF) != (i.hash & 0x01FFFFFFFFFFFFFF)
+        pair = [other, i]
+      end
+      seen[key] ||= i
+      i += 1
+    end
+
+    metric.increment(1, shop_id: pair[0])
+    metric.increment(1, shop_id: pair[1])
+
+    cache = metric.instance_variable_get(:@tag_combination_cache)
+    assert_equal(2, cache.size)
+
+    collision_metric = @sink.datagrams.find do |datagram|
+      datagram.name == "test.statsd_instrument.compiled_metric.hash_collision_detected"
+    end
+    assert_nil(collision_metric, "Expected no hash collision metric for 57-bit-distinct keys")
+  end
+
+  def test_cache_key_computation_does_not_allocate
+    skip_on_jruby("JRuby does not support GC.stat(:total_allocated_objects)")
+
+    single_tag_metric = Class.new(StatsD::Instrument::CompiledMetric::Counter) do
+      define(name: "foo.single", tags: { a: String })
+    end
+    multi_tag_metric = Class.new(StatsD::Instrument::CompiledMetric::Counter) do
+      define(name: "foo.multi", tags: { a: String, b: String, c: String, d: String, e: String, f: String, g: String, h: String })
+    end
+
+    single_call = -> { single_tag_metric.increment(1, a: "a") }
+    multi_call = -> { multi_tag_metric.increment(1, a: "a", b: "b", c: "c", d: "d", e: "e", f: "f", g: "g", h: "h") }
+    measure = ->(callable) { count_allocations { 10.times { callable.call } } }
+    [single_call, multi_call].each { |c| measure.call(c) }
+
+    # With frozen string literals, the rotate-left + XOR chain is the only per-tag
+    # work on a cache hit. A cache key wide enough to escape Fixnum range would
+    # allocate one heap integer per chain step and show up as extra allocations
+    # for the multi-tag metric.
+    assert_equal(
+      measure.call(single_call),
+      measure.call(multi_call),
+      "Expected per-tag cache-key computation to allocate nothing",
+    )
+  end
+
+  private
+
+  def count_allocations
+    GC.start
+    before = GC.stat(:total_allocated_objects)
+    yield
+    GC.stat(:total_allocated_objects) - before
+  end
+
   def test_handles_default_tags_as_array
     StatsD.singleton_client = StatsD::Instrument::Client.new(
       sink: @sink,

data/test/rubocop/singleton_configuration_test.rb

@@ -18,7 +18,10 @@ module Rubocop
 
     def test_offense_statsd_prefix
       assert_offense('StatsD.prefix = "foo"')
+      # The interpolation is intentionally literal source passed to the cop under test.
+      # rubocop:disable Lint/InterpolationCheck
       assert_offense('"#{StatsD.prefix}.foo"')
+      # rubocop:enable Lint/InterpolationCheck
     end
 
     def test_offense_statsd_default_tags

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: statsd-instrument
 version: !ruby/object:Gem::Version
-  version: 3.11.0
+  version: 3.11.2
 platform: ruby
 authors:
 - Jesse Storimer
@@ -135,7 +135,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.9
+rubygems_version: 4.0.18
 specification_version: 4
 summary: A StatsD client for Ruby apps
 test_files: