statsd-instrument 3.9.10 → 3.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d6bb36fa3a5254a64999335275c5515579febf9c2d92d35522fee1d883d0f56d
-  data.tar.gz: 69c96c4f5abaa8cae674a85e562a41ab3bbcf05cde0ef523c934d86846103658
+  metadata.gz: 4768c067e6be6c254db4557916f0cfecb4b86cb318bc8c220acfaf67e9930029
+  data.tar.gz: 756ea941dc116e8c4ed95f1d6af1060ca1d524271b13cb5a418772fc7952519b
 SHA512:
-  metadata.gz: 321874ee863b3f66e1eb7abbace4e734af55dc86fbd11aa8e009e398c54414f47d91447cdcbf08fb93bd4f7b1d88d58a16217093e9f8978b400187ed513dbd74
-  data.tar.gz: 9f34eed89dee5efc7d9a3d800c1d6824b0396054782694d708dab40180e0c3b423f6a0c2337b995bf6f58d72ae750bed564cf02bb5e13fc08518540e443be400
+  metadata.gz: aab58b7242348378b11869da05bcf548c434b63a5f9155a904532b186e5a420e44891c30b1e551d0627359da05fad9c50414876d6eb57baba5883b7990f1d127
+  data.tar.gz: aece18a4af6ef4a14fbb437a91b3dcbd84b7a876a815210d3fb1e207b4851bc8803531589f1b985563da5a31e93e2724d1cc18d2d063742d7dc91ccbeb5abf60

data/.github/workflows/tests.yml

@@ -9,7 +9,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: ['2.7', '3.0', '3.1', '3.2', '3.3', 'ruby-head', 'jruby-9.4.9.0', 'truffleruby-22.3.1']
+        ruby: ['2.7', '3.0', '3.1', '3.2', '3.3', 'ruby-head', 'jruby-9.4.9.0', 'truffleruby-25.0.0']
 
     steps:
     - uses: actions/checkout@v4

data/CHANGELOG.md

@@ -6,6 +6,19 @@ section below.
 
 ## Unreleased changes
 
+## Version 3.10.0
+
+- [#416](https://github.com/Shopify/statsd-instrument/pull/416) - Fix missing `metric_prefix` in Aggregator finalizer, causing metrics to lose their prefix when flushed during GC.
+- [#415](https://github.com/Shopify/statsd-instrument/pull/415) - Fix `sample_rate` being ignored when aggregation is enabled. Previously, `increment`, `measure`, and `histogram` calls would bypass sampling entirely when aggregation was enabled, causing metrics to be emitted at 100% rate regardless of the configured sample rate.
+- [#411](https://github.com/Shopify/statsd-instrument/pull/403) - Add `CompiledMetric::Gauge` as the second metric type to support pre-compiled metric datagrams. It can be used as a replacement over standard `StatsD.gauge`.
+- Add support for `nil` tag values in `CompiledMetric` dynamic tags (converted to empty string).
+- [#407](https://github.com/Shopify/statsd-instrument/pull/407) - Add support for `Symbol` and `:Boolean` types in `CompiledMetric` dynamic tags, allowing symbol and boolean values to be used as tag values.
+- [#403](https://github.com/Shopify/statsd-instrument/pull/403) - Add `CompiledMetric::Distribution` as the second metric type to support pre-compiled metric datagrams. It can be used as a replacement over standard `StatsD.distribution`.
+- [#404](https://github.com/Shopify/statsd-instrument/pull/404) - Fix bug when using aggregation; finalizer was called with the wrong values.
+- [#401](https://github.com/Shopify/statsd-instrument/pull/401), [#402](https://github.com/Shopify/statsd-instrument/pull/402) - Add `CompiledMetric` pattern for high-performance metric emission. Pre-compiles metric datagrams at definition time with tag caching for 2-7x performance improvement over standard `StatsD.increment`. Currently supports Counter metrics only.
+- [#399](https://github.com/Shopify/statsd-instrument/pull/399) - Release the mutex protecting the Aggregator's internal
+  state when flushing metrics to avoid stalling other threads trying to emit metrics.
+
 ## Version 3.9.10
 
 - [#398](https://github.com/Shopify/statsd-instrument/pull/398) - Fix metrics not being sent from signal trap contexts when aggregation is enabled.

data/lib/statsd/instrument/aggregator.rb

@@ -11,7 +11,7 @@ module StatsD
         @no_prefix = no_prefix
         @type = type
         @sample_rate = sample_rate
-        @hash = [@name, @tags, @no_prefix, @type].hash
+        @hash = [@name, @tags, @no_prefix, @type, @sample_rate].hash
       end
 
       def ==(other)
@@ -19,7 +19,8 @@ module StatsD
           @name == other.name &&
           @tags == other.tags &&
           @no_prefix == other.no_prefix &&
-          @type == other.type
+          @type == other.type &&
+          @sample_rate == other.sample_rate
       end
       alias_method :eql?, :==
     end
@@ -33,15 +34,31 @@ module StatsD
       MEASURE = :ms
       HISTOGRAM = :h
       GAUGE = :g
+
+      Finalizer = Struct.new(:aggregation_state, :sink, :datagram_builders, :datagram_builder_class, :default_tags, :metric_prefix)
+
       private_constant :COUNT, :DISTRIBUTION, :MEASURE, :HISTOGRAM, :GAUGE, :CONST_SAMPLE_RATE
 
       class << self
-        def finalize(aggregation_state, sink, datagram_builders, datagram_builder_class, default_tags)
+        def finalize(finalizer)
+          # The finalizer can be called in trap context, which means we cannot use mutexes here.
           proc do
+            aggregation_state = finalizer.aggregation_state
+            sink = finalizer.sink
+            datagram_builders = finalizer.datagram_builders
+            datagram_builder_class = finalizer.datagram_builder_class
+            default_tags = finalizer.default_tags
+            metric_prefix = finalizer.metric_prefix
+
             aggregation_state.each do |key, agg_value|
+              if key.is_a?(StatsD::Instrument::CompiledMetric::PrecompiledDatagram)
+                sink << key.to_datagram(agg_value)
+                next
+              end
+
               no_prefix = key.no_prefix
               datagram_builders[no_prefix] ||= datagram_builder_class.new(
-                prefix: no_prefix ? nil : @metric_prefix,
+                prefix: no_prefix ? nil : metric_prefix,
                 default_tags: default_tags,
               )
               case key.type
@@ -49,7 +66,7 @@ module StatsD
                 sink << datagram_builders[no_prefix].c(
                   key.name,
                   agg_value,
-                  CONST_SAMPLE_RATE,
+                  key.sample_rate,
                   key.tags,
                 )
               when DISTRIBUTION, MEASURE, HISTOGRAM
@@ -71,7 +88,6 @@ module StatsD
                 StatsD.logger.error { "[#{self.class.name}] Unknown aggregation type: #{key.type}" }
               end
             end
-            aggregation_state.clear
           end
         end
       end
@@ -101,16 +117,28 @@ module StatsD
         @max_values = max_values
 
         # Mutex protects the aggregation_state and flush_thread from concurrent access
-        @mutex = Mutex.new
+        @aggregation_state_mutex = Mutex.new
+        # Mutex protects do_flush to prevent flushing to the sink concurrently
+        @flush_mutex = Mutex.new
+
         @aggregation_state = {}
 
         @pid = Process.pid
         @flush_interval = flush_interval
         start_flush_thread
 
+        @finalizer = Finalizer.new(
+          @aggregation_state,
+          @sink,
+          @datagram_builders,
+          @datagram_builder_class,
+          @default_tags,
+          @metric_prefix,
+        )
+
         ObjectSpace.define_finalizer(
           self,
-          self.class.finalize(@aggregation_state, @sink, @datagram_builders, @datagram_builder_class, @default_tags),
+          self.class.finalize(@finalizer),
         )
       end
 
@@ -120,21 +148,82 @@ module StatsD
       # @param tags [Hash{String, Symbol => String},Array<String>] The tags to attach to the counter.
       # @param no_prefix [Boolean] If true, the metric will not be prefixed.
       # @return [void]
-      def increment(name, value = 1, tags: [], no_prefix: false)
+      def increment(name, value = 1, tags: [], no_prefix: false, sample_rate: CONST_SAMPLE_RATE)
         unless thread_healthcheck
-          @sink << datagram_builder(no_prefix: no_prefix).c(name, value, CONST_SAMPLE_RATE, tags)
+          @sink << datagram_builder(no_prefix: no_prefix).c(name, value, sample_rate, tags)
           return
         end
 
         tags = tags_sorted(tags)
-        key = packet_key(name, tags, no_prefix, COUNT)
+        key = packet_key(name, tags, no_prefix, COUNT, sample_rate: sample_rate)
 
-        @mutex.synchronize do
+        @aggregation_state_mutex.synchronize do
           @aggregation_state[key] ||= 0
           @aggregation_state[key] += value
         end
       end
 
+      # Aggregates a precompiled metric that can be combined into a single scalar for later flushing.
+      # @param precompiled_datagram [StatsD::Instrument::CompiledMetric::PrecompiledDatagram]
+      #   The precompiled metric datagram, with the tag values already "filled-in".
+      # @param value [Numeric] The value to aggregate
+      # @return [void]
+      def aggregate_precompiled_increment_metric(precompiled_datagram, value = 1)
+        unless thread_healthcheck
+          # Fallback: emit directly if thread is unhealthy
+          @sink << precompiled_datagram.to_datagram(value)
+          return
+        end
+
+        @aggregation_state_mutex.synchronize do
+          @aggregation_state[precompiled_datagram] ||= 0
+          @aggregation_state[precompiled_datagram] += value
+        end
+      end
+
+      # Aggregates a precompiled metric that can be packed into a single datagram for later flushing.
+      # @param precompiled_datagram [StatsD::Instrument::CompiledMetric::PrecompiledDatagram]
+      #   The precompiled metric datagram, with the tag values already "filled-in".
+      # @param value [Numeric] The value to aggregate
+      # @return [void]
+      def aggregate_precompiled_timing_metric(precompiled_datagram, value = 1)
+        unless thread_healthcheck
+          # Fallback: emit directly if thread is unhealthy
+          @sink << precompiled_datagram.to_datagram(value)
+          return
+        end
+
+        aggregation_state = nil
+
+        @aggregation_state_mutex.synchronize do
+          values = @aggregation_state[precompiled_datagram] ||= []
+          if values.size + 1 >= @max_values
+            aggregation_state = @aggregation_state
+            new_aggregation_state
+          end
+          values << value
+        end
+
+        do_flush(aggregation_state) if aggregation_state
+      end
+
+      # Aggregates a precompiled metric that can be combined into a single scalar for later flushing.
+      # @param precompiled_datagram [StatsD::Instrument::CompiledMetric::PrecompiledDatagram]
+      #   The precompiled metric datagram, with the tag values already "filled-in".
+      # @param value [Numeric] The value to aggregate
+      # @return [void]
+      def aggregate_precompiled_gauge_metric(precompiled_datagram, value)
+        unless thread_healthcheck
+          # Fallback: emit directly if thread is unhealthy
+          @sink << precompiled_datagram.to_datagram(value)
+          return
+        end
+
+        @aggregation_state_mutex.synchronize do
+          @aggregation_state[precompiled_datagram] = value
+        end
+      end
+
       def aggregate_timing(name, value, tags: [], no_prefix: false, type: DISTRIBUTION, sample_rate: CONST_SAMPLE_RATE)
         unless thread_healthcheck
           @sink << datagram_builder(no_prefix: no_prefix).timing_value_packed(
@@ -146,13 +235,18 @@ module StatsD
         tags = tags_sorted(tags)
         key = packet_key(name, tags, no_prefix, type, sample_rate: sample_rate)
 
-        @mutex.synchronize do
+        aggregation_state = nil
+
+        @aggregation_state_mutex.synchronize do
           values = @aggregation_state[key] ||= []
           if values.size + 1 >= @max_values
-            do_flush
+            aggregation_state = @aggregation_state
+            new_aggregation_state
           end
           values << value
         end
+
+        do_flush(aggregation_state) if aggregation_state
       end
 
       def gauge(name, value, tags: [], no_prefix: false)
@@ -164,52 +258,69 @@ module StatsD
         tags = tags_sorted(tags)
         key = packet_key(name, tags, no_prefix, GAUGE)
 
-        @mutex.synchronize do
+        @aggregation_state_mutex.synchronize do
           @aggregation_state[key] = value
         end
       end
 
       def flush
-        @mutex.synchronize { do_flush }
+        state = nil
+        @aggregation_state_mutex.synchronize do
+          state = @aggregation_state
+          new_aggregation_state
+        end
+
+        do_flush(state)
       end
 
       private
 
       EMPTY_ARRAY = [].freeze
 
+      def new_aggregation_state
+        @aggregation_state = {}
+        @finalizer.aggregation_state = @aggregation_state
+      end
+
       # Flushes the aggregated metrics to the sink.
       # Iterates over the aggregation state and sends each metric to the sink.
       # If you change this function, you need to update the logic in the finalizer as well.
-      def do_flush
-        @aggregation_state.each do |key, value|
-          case key.type
-          when COUNT
-            @sink << datagram_builder(no_prefix: key.no_prefix).c(
-              key.name,
-              value,
-              CONST_SAMPLE_RATE,
-              key.tags,
-            )
-          when DISTRIBUTION, MEASURE, HISTOGRAM
-            @sink << datagram_builder(no_prefix: key.no_prefix).timing_value_packed(
-              key.name,
-              key.type.to_s,
-              value,
-              key.sample_rate,
-              key.tags,
-            )
-          when GAUGE
-            @sink << datagram_builder(no_prefix: key.no_prefix).g(
-              key.name,
-              value,
-              CONST_SAMPLE_RATE,
-              key.tags,
-            )
-          else
-            StatsD.logger.error { "[#{self.class.name}] Unknown aggregation type: #{key.type}" }
+      def do_flush(aggregation_state)
+        @flush_mutex.synchronize do
+          aggregation_state.each do |key, value|
+            if key.is_a?(StatsD::Instrument::CompiledMetric::PrecompiledDatagram)
+              @sink << key.to_datagram(value)
+              next
+            end
+
+            case key.type
+            when COUNT
+              @sink << datagram_builder(no_prefix: key.no_prefix).c(
+                key.name,
+                value,
+                key.sample_rate,
+                key.tags,
+              )
+            when DISTRIBUTION, MEASURE, HISTOGRAM
+              @sink << datagram_builder(no_prefix: key.no_prefix).timing_value_packed(
+                key.name,
+                key.type.to_s,
+                value,
+                key.sample_rate,
+                key.tags,
+              )
+            when GAUGE
+              @sink << datagram_builder(no_prefix: key.no_prefix).g(
+                key.name,
+                value,
+                CONST_SAMPLE_RATE,
+                key.tags,
+              )
+            else
+              StatsD.logger.error { "[#{self.class.name}] Unknown aggregation type: #{key.type}" }
+            end
           end
         end
-        @aggregation_state.clear
       end
 
       def tags_sorted(tags)
@@ -255,7 +366,7 @@ module StatsD
       end
 
       def thread_healthcheck
-        @mutex.synchronize do
+        @aggregation_state_mutex.synchronize do
           unless @flush_thread&.alive?
             # The main thread is dead, fallback to direct writes
             return false unless Thread.main.alive?

data/lib/statsd/instrument/client.rb

@@ -220,14 +220,95 @@ module StatsD
       def increment(name, value = 1, sample_rate: nil, tags: nil, no_prefix: false)
         sample_rate ||= @default_sample_rate
 
+        return StatsD::Instrument::VOID if sample_rate && !sample?(sample_rate)
+
+        if @enable_aggregation
+          @aggregator.increment(name, value, tags: tags, no_prefix: no_prefix, sample_rate: sample_rate)
+        else
+          emit(datagram_builder(no_prefix: no_prefix).c(name, value, sample_rate, tags))
+        end
+
+        StatsD::Instrument::VOID
+      end
+
+      # Emits a precompiled metric for high-performance use cases.
+      # This method is used by {StatsD::Instrument::CompiledMetric::Counter} to emit metrics
+      # with minimal allocations.
+      #
+      # @param precompiled_datagram [StatsD::Instrument::CompiledMetric::PrecompiledDatagram]
+      #   The precompiled metric datagram
+      # @param value [Numeric] The metric value
+      # @return [void]
+      # @api private
+      def emit_precompiled_increment_metric(precompiled_datagram, value)
+        sample_rate = precompiled_datagram.sample_rate
+
         if @enable_aggregation
-          @aggregator.increment(name, value, tags: tags, no_prefix: no_prefix)
+          # Sampling decision is done at the definition of a compiled metric, see StatsD::Instrument::CompiledMetric#define
+          if sample_rate.nil? || sample?(sample_rate)
+            @aggregator.aggregate_precompiled_increment_metric(precompiled_datagram, value)
+          end
           return StatsD::Instrument::VOID
         end
 
         if sample_rate.nil? || sample?(sample_rate)
-          emit(datagram_builder(no_prefix: no_prefix).c(name, value, sample_rate, tags))
+          emit(precompiled_datagram.to_datagram(value))
         end
+
+        StatsD::Instrument::VOID
+      end
+
+      # Emits a precompiled metric for high-performance use cases.
+      # This method is used by {StatsD::Instrument::CompiledMetric::Distribution} to emit metrics
+      # with minimal allocations.
+      #
+      # @param precompiled_datagram [StatsD::Instrument::CompiledMetric::PrecompiledDatagram]
+      #   The precompiled metric datagram
+      # @param value [Numeric] The metric value
+      # @return [void]
+      # @api private
+      def emit_precompiled_distribution_metric(precompiled_datagram, value)
+        sample_rate = precompiled_datagram.sample_rate
+
+        if @enable_aggregation
+          # Sampling decision is done at the definition of a compiled metric, see StatsD::Instrument::CompiledMetric.define
+          if sample_rate.nil? || sample?(sample_rate)
+            @aggregator.aggregate_precompiled_timing_metric(precompiled_datagram, value)
+          end
+          return StatsD::Instrument::VOID
+        end
+
+        if sample_rate.nil? || sample?(sample_rate)
+          emit(precompiled_datagram.to_datagram(value))
+        end
+
+        StatsD::Instrument::VOID
+      end
+
+      # Emits a precompiled metric for high-performance use cases.
+      # This method is used by {StatsD::Instrument::CompiledMetric::Gauge} to emit metrics
+      # with minimal allocations.
+      #
+      # @param precompiled_datagram [StatsD::Instrument::CompiledMetric::PrecompiledDatagram]
+      #   The precompiled metric datagram
+      # @param value [Numeric] The metric value
+      # @return [void]
+      # @api private
+      def emit_precompiled_gauge_metric(precompiled_datagram, value)
+        sample_rate = precompiled_datagram.sample_rate
+
+        if @enable_aggregation
+          # Sampling decision is done at the definition of a compiled metric, see StatsD::Instrument::CompiledMetric#define
+          if sample_rate.nil? || sample?(sample_rate)
+            @aggregator.aggregate_precompiled_gauge_metric(precompiled_datagram, value)
+          end
+          return StatsD::Instrument::VOID
+        end
+
+        if sample_rate.nil? || sample?(sample_rate)
+          emit(precompiled_datagram.to_datagram(value))
+        end
+
         StatsD::Instrument::VOID
       end
 
@@ -257,7 +338,7 @@ module StatsD
         end
 
         if @enable_aggregation
-          @aggregator.aggregate_timing(name, value, tags: tags, no_prefix: no_prefix, type: :ms)
+          @aggregator.aggregate_timing(name, value, tags: tags, no_prefix: no_prefix, type: :ms, sample_rate: sample_rate)
           return StatsD::Instrument::VOID
         end
         emit(datagram_builder(no_prefix: no_prefix).ms(name, value, sample_rate, tags))
@@ -367,7 +448,7 @@ module StatsD
         end
 
         if @enable_aggregation
-          @aggregator.aggregate_timing(name, value, tags: tags, no_prefix: no_prefix, type: :h)
+          @aggregator.aggregate_timing(name, value, tags: tags, no_prefix: no_prefix, type: :h, sample_rate: sample_rate)
           return StatsD::Instrument::VOID
         end