fast-prometheus 0.1.0 → 0.3.0

data/lib/fast/prometheus/otlp/mapper.rb

@@ -1,17 +1,14 @@
 # frozen_string_literal: true
 
 require "fast/prometheus"
-
-$LOAD_PATH.unshift(File.expand_path("pb", __dir__)) unless $LOAD_PATH.include?(File.expand_path("pb", __dir__))
-
-require "opentelemetry/proto/collector/metrics/v1/metrics_service_pb"
+require_relative "proto"
 
 module Fast
   module Prometheus
     module OTLP
-      # Maps a Snapshot to an ExportMetricsServiceRequest.
+      # Maps a Snapshot to a Proto::ExportMetricsServiceRequest.
       class Mapper
-        CUMULATIVE = Opentelemetry::Proto::Metrics::V1::AggregationTemporality::AGGREGATION_TEMPORALITY_CUMULATIVE
+        CUMULATIVE = :AGGREGATION_TEMPORALITY_CUMULATIVE
         private_constant :CUMULATIVE
 
         def initialize(resource_attributes: {}, start_time: Time.now)
@@ -20,15 +17,15 @@ module Fast
         end
 
         def request(snapshot)
-          Opentelemetry::Proto::Collector::Metrics::V1::ExportMetricsServiceRequest.new(
+          Proto::ExportMetricsServiceRequest.new(
             resource_metrics: [
-              Opentelemetry::Proto::Metrics::V1::ResourceMetrics.new(
-                resource: Opentelemetry::Proto::Resource::V1::Resource.new(
+              Proto::ResourceMetrics.new(
+                resource: Proto::Resource.new(
                   attributes: @resource_attributes
                 ),
                 scope_metrics: [
-                  Opentelemetry::Proto::Metrics::V1::ScopeMetrics.new(
-                    scope: Opentelemetry::Proto::Common::V1::InstrumentationScope.new(
+                  Proto::ScopeMetrics.new(
+                    scope: Proto::InstrumentationScope.new(
                       name: "fast-prometheus",
                       version: Fast::Prometheus::VERSION
                     ),
@@ -45,7 +42,7 @@ module Fast
         def build_metric(metric_snapshot, taken_at)
           time_unix_nano = (taken_at.to_f * 1_000_000_000).to_i
 
-          Opentelemetry::Proto::Metrics::V1::Metric.new(
+          Proto::Metric.new(
             name: metric_snapshot.name.to_s,
             description: metric_snapshot.docstring,
             **build_data(metric_snapshot, time_unix_nano)
@@ -56,9 +53,9 @@ module Fast
           case metric_snapshot.type
           when :counter
             {
-              sum: Opentelemetry::Proto::Metrics::V1::Sum.new(
-                data_points: metric_snapshot.series.map do |series|
-                  number_data_point(series, time_unix_nano)
+              sum: Proto::Sum.new(
+                data_points: data_points(metric_snapshot) do |attributes, value|
+                  number_data_point(attributes, value, time_unix_nano)
                 end,
                 aggregation_temporality: CUMULATIVE,
                 is_monotonic: true
@@ -66,34 +63,34 @@ module Fast
             }
           when :gauge
             {
-              gauge: Opentelemetry::Proto::Metrics::V1::Gauge.new(
-                data_points: metric_snapshot.series.map do |series|
-                  number_data_point(series, time_unix_nano)
+              gauge: Proto::Gauge.new(
+                data_points: data_points(metric_snapshot) do |attributes, value|
+                  number_data_point(attributes, value, time_unix_nano)
                 end
               )
             }
           when :histogram
             {
-              histogram: Opentelemetry::Proto::Metrics::V1::Histogram.new(
-                data_points: metric_snapshot.series.map do |series|
-                  histogram_data_point(series, time_unix_nano)
+              histogram: Proto::Histogram.new(
+                data_points: data_points(metric_snapshot) do |attributes, value|
+                  histogram_data_point(attributes, value, time_unix_nano)
                 end,
                 aggregation_temporality: CUMULATIVE
               )
             }
           when :summary
             {
-              summary: Opentelemetry::Proto::Metrics::V1::Summary.new(
-                data_points: metric_snapshot.series.map do |series|
-                  summary_data_point(series, time_unix_nano)
+              summary: Proto::Summary.new(
+                data_points: data_points(metric_snapshot) do |attributes, value|
+                  summary_data_point(attributes, value, time_unix_nano)
                 end
               )
             }
           when :native_histogram
             {
-              exponential_histogram: Opentelemetry::Proto::Metrics::V1::ExponentialHistogram.new(
-                data_points: metric_snapshot.series.map do |series|
-                  exponential_histogram_data_point(series, time_unix_nano)
+              exponential_histogram: Proto::ExponentialHistogram.new(
+                data_points: data_points(metric_snapshot) do |attributes, value|
+                  exponential_histogram_data_point(attributes, value, time_unix_nano)
                 end,
                 aggregation_temporality: CUMULATIVE
               )
@@ -101,25 +98,30 @@ module Fast
           end
         end
 
-        def number_data_point(series, time_unix_nano)
-          Opentelemetry::Proto::Metrics::V1::NumberDataPoint.new(
-            attributes: build_attributes(series.labels),
+        # One data point per series: the block gets the series' OTLP
+        # attributes and its snapshot value.
+        def data_points(metric_snapshot)
+          metric_snapshot.series.map { |values, value| yield build_attributes(metric_snapshot.labels(values)), value }
+        end
+
+        def number_data_point(attributes, value, time_unix_nano)
+          Proto::NumberDataPoint.new(
+            attributes: attributes,
             start_time_unix_nano: @start_time_unix_nano,
             time_unix_nano: time_unix_nano,
-            as_double: series.value.to_f
+            as_double: value.to_f
           )
         end
 
-        def histogram_data_point(series, time_unix_nano)
-          nv = series.value
-          bounds, counts = non_cumulative_buckets(nv.cumulative_buckets)
+        def histogram_data_point(attributes, value, time_unix_nano)
+          bounds, counts = non_cumulative_buckets(value.cumulative_buckets)
 
-          Opentelemetry::Proto::Metrics::V1::HistogramDataPoint.new(
-            attributes: build_attributes(series.labels),
+          Proto::HistogramDataPoint.new(
+            attributes: attributes,
             start_time_unix_nano: @start_time_unix_nano,
             time_unix_nano: time_unix_nano,
-            count: nv.count,
-            sum: nv.sum,
+            count: value.count,
+            sum: value.sum,
             bucket_counts: counts,
             explicit_bounds: bounds
           )
@@ -142,32 +144,28 @@ module Fast
           [bounds, counts]
         end
 
-        def summary_data_point(series, time_unix_nano)
-          sv = series.value
-
-          Opentelemetry::Proto::Metrics::V1::SummaryDataPoint.new(
-            attributes: build_attributes(series.labels),
+        def summary_data_point(attributes, value, time_unix_nano)
+          Proto::SummaryDataPoint.new(
+            attributes: attributes,
             start_time_unix_nano: @start_time_unix_nano,
             time_unix_nano: time_unix_nano,
-            count: sv.count,
-            sum: sv.sum
+            count: value.count,
+            sum: value.sum
           )
         end
 
-        def exponential_histogram_data_point(series, time_unix_nano)
-          nv = series.value
-
-          Opentelemetry::Proto::Metrics::V1::ExponentialHistogramDataPoint.new(
-            attributes: build_attributes(series.labels),
+        def exponential_histogram_data_point(attributes, value, time_unix_nano)
+          Proto::ExponentialHistogramDataPoint.new(
+            attributes: attributes,
             start_time_unix_nano: @start_time_unix_nano,
             time_unix_nano: time_unix_nano,
-            count: nv.count,
-            sum: nv.sum,
-            scale: nv.schema,
-            zero_count: nv.zero_count,
-            zero_threshold: nv.zero_threshold,
-            positive: dense_buckets(nv.positive_buckets),
-            negative: dense_buckets(nv.negative_buckets)
+            count: value.count,
+            sum: value.sum,
+            scale: value.schema,
+            zero_count: value.zero_count,
+            zero_threshold: value.zero_threshold,
+            positive: dense_buckets(value.positive_buckets),
+            negative: dense_buckets(value.negative_buckets)
           )
         end
 
@@ -175,7 +173,7 @@ module Fast
         # OTLP index = prom index - 1. Offset = min OTLP index.
         # Buckets are always sorted (from NativeHistogram::Slot#positive_buckets / #negative_buckets).
         def dense_buckets(buckets)
-          return Opentelemetry::Proto::Metrics::V1::ExponentialHistogramDataPoint::Buckets.new if buckets.empty?
+          return Proto::ExponentialHistogramDataPoint::Buckets.new if buckets.empty?
 
           first_idx, = buckets.first
           last_idx, = buckets.last
@@ -185,7 +183,7 @@ module Fast
           counts = Array.new(length, 0)
           buckets.each { |prom_idx, count| counts[prom_idx - offset - 1] = count }
 
-          Opentelemetry::Proto::Metrics::V1::ExponentialHistogramDataPoint::Buckets.new(
+          Proto::ExponentialHistogramDataPoint::Buckets.new(
             offset: offset,
             bucket_counts: counts
           )
@@ -196,9 +194,9 @@ module Fast
         end
 
         def kv(key, value)
-          Opentelemetry::Proto::Common::V1::KeyValue.new(
+          Proto::KeyValue.new(
             key: key,
-            value: Opentelemetry::Proto::Common::V1::AnyValue.new(string_value: value)
+            value: Proto::AnyValue.new(string_value: value)
           )
         end
       end

data/lib/fast/prometheus/otlp/proto.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+require "fast/protowire"
+
+module Fast
+  module Prometheus
+    module OTLP
+      # The OTLP metrics data model (proto/opentelemetry, proto3), declared
+      # for the wire: what the exporters send and the one response they read.
+      module Proto
+        class AnyValue < Fast::Protowire::Message
+          oneof :value do
+            field :string_value, :string, 1
+            field :bool_value, :bool, 2
+            field :int_value, :int64, 3
+            field :double_value, :double, 4
+            field :bytes_value, :bytes, 7
+          end
+        end
+
+        class KeyValue < Fast::Protowire::Message
+          field :key, :string, 1
+          field :value, AnyValue, 2
+        end
+
+        class InstrumentationScope < Fast::Protowire::Message
+          field :name, :string, 1
+          field :version, :string, 2
+          repeated :attributes, KeyValue, 3
+          field :dropped_attributes_count, :uint32, 4
+        end
+
+        class Resource < Fast::Protowire::Message
+          repeated :attributes, KeyValue, 1
+          field :dropped_attributes_count, :uint32, 2
+        end
+
+        AggregationTemporality = Fast::Protowire::Enum.define(
+          AGGREGATION_TEMPORALITY_UNSPECIFIED: 0,
+          AGGREGATION_TEMPORALITY_DELTA: 1,
+          AGGREGATION_TEMPORALITY_CUMULATIVE: 2
+        )
+
+        class NumberDataPoint < Fast::Protowire::Message
+          field :start_time_unix_nano, :fixed64, 2
+          field :time_unix_nano, :fixed64, 3
+          oneof :value do
+            field :as_double, :double, 4
+            field :as_int, :sfixed64, 6
+          end
+          repeated :attributes, KeyValue, 7
+          field :flags, :uint32, 8
+        end
+
+        class HistogramDataPoint < Fast::Protowire::Message
+          field :start_time_unix_nano, :fixed64, 2
+          field :time_unix_nano, :fixed64, 3
+          field :count, :fixed64, 4
+          optional :sum, :double, 5
+          repeated :bucket_counts, :fixed64, 6
+          repeated :explicit_bounds, :double, 7
+          repeated :attributes, KeyValue, 9
+          field :flags, :uint32, 10
+          optional :min, :double, 11
+          optional :max, :double, 12
+        end
+
+        class SummaryDataPoint < Fast::Protowire::Message
+          class ValueAtQuantile < Fast::Protowire::Message
+            field :quantile, :double, 1
+            field :value, :double, 2
+          end
+
+          field :start_time_unix_nano, :fixed64, 2
+          field :time_unix_nano, :fixed64, 3
+          field :count, :fixed64, 4
+          field :sum, :double, 5
+          repeated :quantile_values, ValueAtQuantile, 6
+          repeated :attributes, KeyValue, 7
+          field :flags, :uint32, 8
+        end
+
+        class ExponentialHistogramDataPoint < Fast::Protowire::Message
+          class Buckets < Fast::Protowire::Message
+            field :offset, :sint32, 1
+            repeated :bucket_counts, :uint64, 2
+          end
+
+          repeated :attributes, KeyValue, 1
+          field :start_time_unix_nano, :fixed64, 2
+          field :time_unix_nano, :fixed64, 3
+          field :count, :fixed64, 4
+          optional :sum, :double, 5
+          field :scale, :sint32, 6
+          field :zero_count, :fixed64, 7
+          field :positive, Buckets, 8
+          field :negative, Buckets, 9
+          field :flags, :uint32, 10
+          optional :min, :double, 12
+          optional :max, :double, 13
+          field :zero_threshold, :double, 14
+        end
+
+        class Gauge < Fast::Protowire::Message
+          repeated :data_points, NumberDataPoint, 1
+        end
+
+        class Sum < Fast::Protowire::Message
+          repeated :data_points, NumberDataPoint, 1
+          field :aggregation_temporality, AggregationTemporality, 2
+          field :is_monotonic, :bool, 3
+        end
+
+        class Histogram < Fast::Protowire::Message
+          repeated :data_points, HistogramDataPoint, 1
+          field :aggregation_temporality, AggregationTemporality, 2
+        end
+
+        class ExponentialHistogram < Fast::Protowire::Message
+          repeated :data_points, ExponentialHistogramDataPoint, 1
+          field :aggregation_temporality, AggregationTemporality, 2
+        end
+
+        class Summary < Fast::Protowire::Message
+          repeated :data_points, SummaryDataPoint, 1
+        end
+
+        class Metric < Fast::Protowire::Message
+          field :name, :string, 1
+          field :description, :string, 2
+          field :unit, :string, 3
+          oneof :data do
+            field :gauge, Gauge, 5
+            field :sum, Sum, 7
+            field :histogram, Histogram, 9
+            field :exponential_histogram, ExponentialHistogram, 10
+            field :summary, Summary, 11
+          end
+          repeated :metadata, KeyValue, 12
+        end
+
+        class ScopeMetrics < Fast::Protowire::Message
+          field :scope, InstrumentationScope, 1
+          repeated :metrics, Metric, 2
+          field :schema_url, :string, 3
+        end
+
+        class ResourceMetrics < Fast::Protowire::Message
+          field :resource, Resource, 1
+          repeated :scope_metrics, ScopeMetrics, 2
+          field :schema_url, :string, 3
+        end
+
+        class ExportMetricsServiceRequest < Fast::Protowire::Message
+          repeated :resource_metrics, ResourceMetrics, 1
+        end
+
+        class ExportMetricsPartialSuccess < Fast::Protowire::Message
+          field :rejected_data_points, :int64, 1
+          field :error_message, :string, 2
+        end
+
+        class ExportMetricsServiceResponse < Fast::Protowire::Message
+          field :partial_success, ExportMetricsPartialSuccess, 1
+        end
+      end
+    end
+  end
+end

data/lib/fast/prometheus/otlp/service_interface.rb

@@ -2,7 +2,7 @@
 
 require "fast/prometheus"
 require "protocol/grpc/interface"
-require_relative "mapper"
+require_relative "proto"
 
 module Fast
   module Prometheus
@@ -10,8 +10,8 @@ module Fast
       # Protocol::GRPC::Interface for opentelemetry.proto.collector.metrics.v1.MetricsService.
       class MetricsServiceInterface < Protocol::GRPC::Interface
         rpc :Export,
-            request_class: Opentelemetry::Proto::Collector::Metrics::V1::ExportMetricsServiceRequest,
-            response_class: Opentelemetry::Proto::Collector::Metrics::V1::ExportMetricsServiceResponse
+            request_class: Proto::ExportMetricsServiceRequest,
+            response_class: Proto::ExportMetricsServiceResponse
       end
     end
   end

data/lib/fast/prometheus/registry.rb

@@ -28,14 +28,19 @@ module Fast
         end
       end
 
-      # Remove a metric by name.
+      # Remove a metric by name (String or Symbol).
       def unregister(name)
-        @lock.synchronize { @by_name.delete(name) }
+        @lock.synchronize { @by_name.delete(normalize(name)) }
       end
 
-      # Look up a metric by name, or nil.
+      # Look up a metric by name (String or Symbol), or nil.
       def get(name)
-        @lock.synchronize { @by_name[name] }
+        @lock.synchronize { @by_name[normalize(name)] }
+      end
+
+      # True iff a metric is registered under +name+ (String or Symbol).
+      def exist?(name)
+        @lock.synchronize { @by_name.key?(normalize(name)) }
       end
 
       # Atomic fetch-or-register: returns the metric already registered under
@@ -45,15 +50,16 @@ module Fast
       # DuplicateMetric. Raises ArgumentError if the block's metric is not
       # named +name+.
       def fetch_or_register(name)
+        key = normalize(name)
         @lock.synchronize do
-          next @by_name[name] if @by_name.key?(name)
+          next @by_name[key] if @by_name.key?(key)
 
           metric = yield
-          unless metric.name == name
-            raise ArgumentError, "fetch_or_register(#{name.inspect}) block built #{metric.name.inspect}"
+          unless metric.name == key
+            raise ArgumentError, "fetch_or_register(#{key.inspect}) block built #{metric.name.inspect}"
           end
 
-          @by_name[name] = metric
+          @by_name[key] = metric
         end
       end
 
@@ -93,6 +99,12 @@ module Fast
       def collect
         Snapshot.of(metrics)
       end
+
+      private
+
+      def normalize(name)
+        name.to_s.to_sym
+      end
     end
 
     @registry_lock = Mutex.new
@@ -105,7 +117,7 @@ module Fast
     end
 
     def self.registry=(registry)
-      @registry = registry
+      @registry_lock.synchronize { @registry = registry }
     end
   end
 end

data/lib/fast/prometheus/snapshot.rb

@@ -2,66 +2,40 @@
 
 module Fast
   module Prometheus
-    # Immutable value types for snapshots. Deep-frozen Data types.
-
-    HistogramValue = Data.define(:sum, :count, :cumulative_buckets)
-
-    NativeHistogramValue = Data.define(:schema, :zero_threshold, :zero_count, :sum, :count,
-                                       :positive_buckets, :negative_buckets)
-
-    Series = Data.define(:labels, :value)
-
-    SummaryValue = Data.define(:sum, :count)
+    # Immutable value types for snapshots.
+    #
+    # A MetricSnapshot's +series+ is the metric's store as it stood at one
+    # instant: a frozen Hash from each series' label values (an Array in
+    # +label_names+ order, the store's own key) to its value. Counter and
+    # gauge values are Floats; the other types' values are the frozen
+    # Structs below, built from the live slot while the store was locked.
+    # Nothing per series is copied beyond that, so a snapshot of any size
+    # costs one Hash per metric.
+    #
+    # The value shapes are Structs frozen where they are built, not Data:
+    # Data.new allocates two objects besides the instance, and a scrape
+    # builds one of these per histogram, summary or native histogram series.
+    # (sum and count shadow Enumerable's on the Structs, as they would on Data.)
+
+    HistogramValue = Struct.new(:sum, :count, :cumulative_buckets) # rubocop:disable Lint/StructNewOverride
+
+    NativeHistogramValue = Struct.new(:schema, :zero_threshold, :zero_count, :sum, :count, # rubocop:disable Lint/StructNewOverride
+                                      :positive_buckets, :negative_buckets)
+
+    SummaryValue = Struct.new(:sum, :count) # rubocop:disable Lint/StructNewOverride
 
     MetricSnapshot = Data.define(:name, :docstring, :type, :label_names, :series) do
-      # Build a MetricSnapshot from a live metric. Reads only public readers
-      # and copies all mutable structures. The whole build runs under the
-      # metric's lock so no series is observed mid-mutation and no writer
-      # interleaves between series.
+      # Build a MetricSnapshot from a live metric. +series+ is the metric's
+      # store copied and its slots frozen under the metric's lock (see
+      # Metric#snapshot_values), so no series is observed mid-mutation and
+      # no writer interleaves between series.
       def self.of(metric)
-        metric.synchronize do
-          new(
-            metric.name,
-            metric.docstring,
-            metric.type,
-            metric.label_names.dup.freeze,
-            build_series(metric).freeze
-          )
-        end
-      end
-
-      def self.build_series(metric)
-        metric.values.map do |labels, value|
-          Series.new(
-            labels: labels.dup.freeze,
-            value: build_value(metric.type, value)
-          )
-        end
+        new(metric.name, metric.docstring, metric.type, metric.labels.dup.freeze, metric.snapshot_values.freeze)
       end
 
-      private_class_method def self.build_value(type, value)
-        case type
-        when :counter, :gauge
-          value
-        when :histogram
-          HistogramValue.new(
-            sum: value.sum,
-            count: value.count,
-            cumulative_buckets: value.cumulative_buckets.map { |pair| pair.dup.freeze }.freeze
-          )
-        when :summary
-          SummaryValue.new(sum: value.sum, count: value.count)
-        when :native_histogram
-          NativeHistogramValue.new(
-            schema: value.schema,
-            zero_threshold: value.zero_threshold,
-            zero_count: value.zero_count,
-            sum: value.sum,
-            count: value.count,
-            positive_buckets: value.positive_buckets.map { |pair| pair.dup.freeze }.freeze,
-            negative_buckets: value.negative_buckets.map { |pair| pair.dup.freeze }.freeze
-          )
-        end
+      # The {name => value} Hash for one series' label values.
+      def labels(values)
+        label_names.zip(values).to_h
       end
     end
 

data/lib/fast/prometheus/store.rb

@@ -17,8 +17,10 @@ module Fast
         @monitor = Monitor.new
       end
 
-      def synchronize(&block)
-        @monitor.synchronize(&block)
+      # Written with yield rather than a captured &block, so taking the lock
+      # allocates nothing.
+      def synchronize
+        @monitor.synchronize { yield } # rubocop:disable Style/ExplicitBlockArgument
       end
 
       def [](key)

data/lib/fast/prometheus/summary.rb

@@ -21,7 +21,7 @@ module Fast
         :summary
       end
 
-      def observe(value, labels: {})
+      def observe(value, labels: NO_LABELS)
         key = resolve(labels)
         store.synchronize do
           slot = store[key] ||= Value.new
@@ -30,9 +30,14 @@ module Fast
         end
       end
 
-      def get(labels: {})
+      # {"count" => Integer, "sum" => Float}. Zero-valued for an unobserved series.
+      def get(labels: NO_LABELS)
         key = resolve(labels)
-        store.synchronize { store[key] }
+        store.synchronize { hash_shape(store[key] || zero_value) }
+      end
+
+      def values
+        series_map { |slot| hash_shape(slot) }
       end
 
       protected
@@ -45,6 +50,20 @@ module Fast
 
         super
       end
+
+      private
+
+      def zero_value
+        Value.new
+      end
+
+      def snapshot_value(slot)
+        SummaryValue.new(slot.sum, slot.count).freeze
+      end
+
+      def hash_shape(slot)
+        { "count" => slot.count, "sum" => slot.sum }
+      end
     end
   end
 end

data/lib/fast/prometheus/version.rb

@@ -2,6 +2,6 @@
 
 module Fast
   module Prometheus
-    VERSION = "0.1.0"
+    VERSION = "0.3.0"
   end
 end