speedshop-cloudwatch 0.2.1 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a4d2d47eedd3f9ad965073f7569d707bee70f2b4bb35f7b7f97f4197d9016b9c
-  data.tar.gz: d2bc25a79ba9893fa45a8a8843949ef42a7e70b5ab054ced8031446148bed927
+  metadata.gz: 6cff6ca0b03ef4fa6214a14bfe89384b72593aec69ec8fb93288460e3b4228a0
+  data.tar.gz: 1505eadbc697749442550a5ad22df30fa545c27302b6ba2b5c62a1a218b9c248
 SHA512:
-  metadata.gz: ae73508f29611fba12c522b3e84d82331026638892d2b540ede54a951a9e5314254c59a1620f5d423c0f592a7017f526233b938873107348231faf0a8db1c4b6
-  data.tar.gz: eb65e33a190b8d4a33686dd2c63f59ba504b0f585a2cb3d730f09274e89809acfa9ba0034caaef272b4f02fe3a0029c1373f03f684c9d3db2e03e4f1e9b8eab2
+  metadata.gz: 986e68cf85c6c4d06c606312838d701e7309f9fb6a20879b23e1353360ae03a29f056c1c8129ac11e0b10cf381226eabeec7a76a160b0555dc1b8f243815d10b
+  data.tar.gz: 76bc9802a6107aa1f153c096be8bfc94897084cd380646657b10a45cb7dbcbf2e7e98e797fc86c0b87d7084628810e9389bf982f3e9126cd69a0d6b1a7687f9a

data/CHANGELOG.md

@@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+
+## [0.2.2] - 2026-09-07
+
+### Changed
+- Preserve metric distributions with CloudWatch `Values` and `Counts` instead of reducing repeated observations to StatisticSets, enabling percentile statistics without changing application configuration.
+
 ## [0.2.1] - 2026-06-09
 
 ### Fixed
@@ -27,6 +34,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - Initial public release.
 
+[Unreleased]: https://github.com/speedshop/speedshop-cloudwatch/compare/v0.2.2...HEAD
+[0.2.2]: https://github.com/speedshop/speedshop-cloudwatch/compare/v0.2.1...v0.2.2
 [0.2.1]: https://github.com/speedshop/speedshop-cloudwatch/compare/v0.2.0...v0.2.1
 [0.2.0]: https://github.com/speedshop/speedshop-cloudwatch/compare/v0.1.0...v0.2.0
 [0.1.0]: https://github.com/speedshop/speedshop-cloudwatch/releases/tag/v0.1.0

data/README.md

@@ -92,6 +92,16 @@ end
 > [!WARNING]
 > Setting `config.interval` to less than 60 seconds automatically enables [high-resolution storage](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/publishingMetrics.html#high-resolution-metrics) (1-second granularity) in CloudWatch, which incurs additional costs.
 
+### Percentiles and distribution statistics
+
+When a metric has multiple observations in one reporting period, the reporter publishes them using CloudWatch `Values` and `Counts`. Repeated values are frequency-encoded, and distributions with more than CloudWatch's limit of 150 unique values are split across multiple metric datums. Requests retain the 20-datum cap and use a conservative serialized-size bound to stay within CloudWatch's uncompressed 1 MiB limit. An individual datum exceeding that bound is logged and discarded without preventing other datums from being sent.
+
+This preserves the observed distribution, allowing CloudWatch to calculate `p50`, `p95`, `p99`, trimmed means, and other distribution statistics in addition to Average, Sum, Minimum, Maximum, and SampleCount. Observations discarded because `config.queue_max_size` was exceeded are not included.
+
+CloudWatch does not provide percentile statistics for metrics containing negative values. Explicit `statistic_values:` input remains supported, but neither it nor historical StatisticSets recovers discarded distributions.
+
+Explicit Yabeda gauge aggregation strategies retain their existing behavior: `most_recent` publishes the last value and `max` publishes the maximum value.
+
 ### Environment Control
 
 **By default, the reporter only runs in production.** The environment is detected from `RAILS_ENV`, `RACK_ENV`, and defaults to `"development"`.
@@ -210,7 +220,7 @@ We report the following metrics:
 QueueLatency - Time job spent waiting in queue before execution (Seconds)
 ```
 
-This metric includes QueueName dimension and is aggregated per interval using CloudWatch StatisticSets.
+This metric includes a QueueName dimension and preserves the per-interval distribution using CloudWatch `Values` and `Counts`.
 
 ## Yabeda
 

data/lib/speedshop/cloudwatch/metrics.rb

@@ -171,8 +171,8 @@ module Speedshop
           name: :QueueLatency,
           unit: "Seconds",
           description: "Time a job spent waiting in the queue before execution " \
-                       "started. Values are aggregated into CloudWatch " \
-                       "StatisticSets per reporting interval.",
+                       "started. Values are frequency-encoded without discarding " \
+                       "the per-interval distribution.",
           source: "Time.now.to_f - job.enqueued_at"
         )
       ]

data/lib/speedshop/cloudwatch/reporter.rb

@@ -1,10 +1,13 @@
 # frozen_string_literal: true
 
 require "singleton"
+require_relative "request_size"
 
 module Speedshop
   module Cloudwatch
     class Reporter
+      MAX_VALUES_PER_DATUM = 150
+
       include Singleton
 
       def initialize
@@ -195,7 +198,10 @@ module Speedshop
             timestamp: m[:timestamp],
             dimensions: m[:dimensions]
           }
-          if m[:statistic_values]
+          if m[:values]
+            datum[:values] = m[:values]
+            datum[:counts] = m[:counts]
+          elsif m[:statistic_values]
             datum[:statistic_values] = m[:statistic_values]
           else
             datum[:value] = m[:value]
@@ -206,91 +212,95 @@ module Speedshop
       end
 
       def send_batches(namespace, metric_data)
-        metric_data.each_slice(20) do |batch|
-          config.client.put_metric_data(namespace: namespace, metric_data: batch)
+        overhead = RequestSize.request_overhead(namespace)
+        batch = []
+        size = overhead
+        metric_data.each do |datum|
+          datum_size = RequestSize.bound(datum)
+          if overhead + datum_size > RequestSize::LIMIT
+            Speedshop::Cloudwatch.log_error("Dropping oversized CloudWatch datum: #{datum[:metric_name]} (request size bound exceeds 1 MiB)")
+            next
+          end
+          if batch.size == 20 || size + datum_size > RequestSize::LIMIT
+            config.client.put_metric_data(namespace: namespace, metric_data: batch)
+            batch = []
+            size = overhead
+          end
+          batch << datum
+          size += datum_size
         end
+        config.client.put_metric_data(namespace: namespace, metric_data: batch) unless batch.empty?
       end
 
       def aggregate_namespace_metrics(ns_metrics)
-        group_metrics(ns_metrics).map { |items| aggregate_group(items) }
+        group_metrics(ns_metrics).flat_map { |items| aggregate_group(items) }
       end
 
       def group_metrics(ns_metrics)
         groups = {}
         ns_metrics.each do |m|
-          key = [m[:metric_name], m[:unit], normalized_dimensions_key(m[:dimensions]), m[:aggregation_strategy]]
+          key = [
+            m[:metric_name],
+            m[:unit],
+            normalized_dimensions_key(m[:dimensions]),
+            m[:aggregation_strategy],
+            m.key?(:statistic_values),
+            timestamp_bucket(m[:timestamp])
+          ]
           (groups[key] ||= []) << m
         end
         groups.values
       end
 
       def aggregate_group(items)
-        return items.first if items.size == 1
+        return items if items.size == 1
 
         strategy = items.first[:aggregation_strategy]
-        return aggregate_most_recent_group(items) if strategy == :most_recent
-        return aggregate_max_group(items) if strategy == :max
+        return [items.last] if strategy == :most_recent
+        return [items.max_by { |item| item[:value].to_f }] if strategy == :max
+        return [merge_statistic_values_group(items)] if items.first[:statistic_values]
 
-        sample_count, sum, minimum, maximum = aggregate_values(items)
-        {
-          metric_name: items.first[:metric_name],
-          unit: items.first[:unit],
-          dimensions: items.first[:dimensions],
-          timestamp: Time.now,
-          statistic_values: build_statistic_values(sample_count, sum, minimum, maximum)
-        }
+        aggregate_distribution_group(items)
       end
 
-      def aggregate_values(items)
-        sample_count = 0.0
-        sum = 0.0
-        minimum = Float::INFINITY
-        maximum = -Float::INFINITY
-
-        items.each do |item|
-          if item[:statistic_values]
-            sv = item[:statistic_values]
-            sc = sv[:sample_count].to_f
-            sample_count += sc
-            sum += sv[:sum].to_f
-            minimum = [minimum, sv[:minimum].to_f].min
-            maximum = [maximum, sv[:maximum].to_f].max
-          elsif item.key?(:value)
-            v = item[:value].to_f
-            sample_count += 1.0
-            sum += v
-            minimum = [minimum, v].min
-            maximum = [maximum, v].max
-          end
+      def aggregate_distribution_group(items)
+        frequencies = items.each_with_object(Hash.new(0)) do |item, counts|
+          counts[item[:value].to_f] += 1
         end
 
-        [sample_count, sum, minimum, maximum]
-      end
-
-      def aggregate_most_recent_group(items)
-        items.last
-      end
-
-      def aggregate_max_group(items)
-        items.max_by { |item| item_value_for_max(item) }
-      end
-
-      def item_value_for_max(item)
-        return item[:statistic_values][:maximum].to_f if item[:statistic_values]
-        return item[:value].to_f if item.key?(:value)
-
-        -Float::INFINITY
+        frequencies.sort_by(&:first).each_slice(MAX_VALUES_PER_DATUM).map do |slice|
+          {
+            metric_name: items.first[:metric_name],
+            unit: items.first[:unit],
+            dimensions: items.first[:dimensions],
+            timestamp: timestamp_bucket(items.first[:timestamp]),
+            values: slice.map(&:first),
+            counts: slice.map(&:last)
+          }
+        end
       end
 
-      def build_statistic_values(sample_count, sum, minimum, maximum)
+      def merge_statistic_values_group(items)
+        statistic_values = items.map { |item| item[:statistic_values] }
         {
-          sample_count: sample_count,
-          sum: sum,
-          minimum: minimum.finite? ? minimum : 0.0,
-          maximum: maximum.finite? ? maximum : 0.0
+          metric_name: items.first[:metric_name],
+          unit: items.first[:unit],
+          dimensions: items.first[:dimensions],
+          timestamp: timestamp_bucket(items.first[:timestamp]),
+          statistic_values: {
+            sample_count: statistic_values.sum { |values| values[:sample_count].to_f },
+            sum: statistic_values.sum { |values| values[:sum].to_f },
+            minimum: statistic_values.map { |values| values[:minimum].to_f }.min,
+            maximum: statistic_values.map { |values| values[:maximum].to_f }.max
+          }
         }
       end
 
+      def timestamp_bucket(timestamp)
+        period = [config.interval.to_i, 1].max
+        Time.at((timestamp.to_f / period).floor * period)
+      end
+
       def normalized_dimensions_key(dims)
         (dims || []).sort_by { |d| d[:name].to_s }.map { |d| "#{d[:name]}=#{d[:value]}" }.join("|")
       end

data/lib/speedshop/cloudwatch/request_size.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Speedshop
+  module Cloudwatch
+    module RequestSize
+      LIMIT = 1024 * 1024
+
+      # Upper bound for uncompressed PutMetricData Query bodies, not Ruby object size.
+      # Every supported leaf's full Query key (including MetricData.member.20,
+      # Dimensions/Values/Counts indices), '=' and '&' fits within 128 bytes.
+      # Percent encoding expands each UTF-8 byte by at most 3. The 64-byte floor
+      # covers SDK numeric/timestamp formatting independently of Ruby's to_s.
+      # Container overhead is deliberately overcounted, including empty lists.
+      def self.bound(value)
+        case value
+        when Hash
+          128 + value.values.sum { |item| bound(item) }
+        when Array
+          128 + value.sum { |item| bound(item) }
+        else
+          128 + 3 * [value.to_s.bytesize, 64].max
+        end
+      end
+
+      def self.request_overhead(namespace)
+        # Action, Version, Namespace key, and separators.
+        512 + bound(namespace)
+      end
+    end
+  end
+end

data/lib/speedshop/cloudwatch/version.rb

@@ -2,6 +2,6 @@
 
 module Speedshop
   module Cloudwatch
-    VERSION = "0.2.1"
+    VERSION = "0.2.2"
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: speedshop-cloudwatch
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors:
 - Nate Berkopec
@@ -82,6 +82,7 @@ files:
 - lib/speedshop/cloudwatch/rack.rb
 - lib/speedshop/cloudwatch/railtie.rb
 - lib/speedshop/cloudwatch/reporter.rb
+- lib/speedshop/cloudwatch/request_size.rb
 - lib/speedshop/cloudwatch/sidekiq.rb
 - lib/speedshop/cloudwatch/version.rb
 - lib/speedshop/cloudwatch/yabeda.rb