prometheus-client 1.0.0 → 4.1.0

data/lib/prometheus/client/data_stores/direct_file_store.rb

@@ -29,7 +29,7 @@ module Prometheus
 
       class DirectFileStore
         class InvalidStoreSettingsError < StandardError; end
-        AGGREGATION_MODES = [MAX = :max, MIN = :min, SUM = :sum, ALL = :all]
+        AGGREGATION_MODES = [MAX = :max, MIN = :min, SUM = :sum, ALL = :all, MOST_RECENT = :most_recent]
         DEFAULT_METRIC_SETTINGS = { aggregation: SUM }
         DEFAULT_GAUGE_SETTINGS = { aggregation: ALL }
 
@@ -45,7 +45,7 @@ module Prometheus
           end
 
           settings = default_settings.merge(metric_settings)
-          validate_metric_settings(settings)
+          validate_metric_settings(metric_type, settings)
 
           MetricStore.new(metric_name: metric_name,
                           store_settings: @store_settings,
@@ -54,7 +54,7 @@ module Prometheus
 
         private
 
-        def validate_metric_settings(metric_settings)
+        def validate_metric_settings(metric_type, metric_settings)
           unless metric_settings.has_key?(:aggregation) &&
             AGGREGATION_MODES.include?(metric_settings[:aggregation])
             raise InvalidStoreSettingsError,
@@ -65,6 +65,11 @@ module Prometheus
             raise InvalidStoreSettingsError,
                   "Only :aggregation setting can be specified"
           end
+
+          if metric_settings[:aggregation] == MOST_RECENT && metric_type != :gauge
+            raise InvalidStoreSettingsError,
+                  "Only :gauge metrics support :most_recent aggregation"
+          end
         end
 
         class MetricStore
@@ -74,6 +79,7 @@ module Prometheus
             @metric_name = metric_name
             @store_settings = store_settings
             @values_aggregation_mode = metric_settings[:aggregation]
+            @store_opened_by_pid = nil
 
             @lock = Monitor.new
           end
@@ -100,10 +106,15 @@ module Prometheus
           end
 
           def increment(labels:, by: 1)
+            if @values_aggregation_mode == DirectFileStore::MOST_RECENT
+              raise InvalidStoreSettingsError,
+                    "The :most_recent aggregation does not support the use of increment"\
+                      "/decrement"
+            end
+
             key = store_key(labels)
             in_process_sync do
-              value = internal_store.read_value(key)
-              internal_store.write_value(key, value + by.to_f)
+              internal_store.increment_value(key, by.to_f)
             end
           end
 
@@ -121,7 +132,7 @@ module Prometheus
             stores_for_metric.each do |file_path|
               begin
                 store = FileMappedDict.new(file_path, true)
-                store.all_values.each do |(labelset_qs, v)|
+                store.all_values.each do |(labelset_qs, v, ts)|
                   # Labels come as a query string, and CGI::parse returns arrays for each key
                   # "foo=bar&x=y" => { "foo" => ["bar"], "x" => ["y"] }
                   # Turn the keys back into symbols, and remove the arrays
@@ -129,7 +140,7 @@ module Prometheus
                     [k.to_sym, vs.first]
                   end.to_h
 
-                  stores_data[label_set] << v
+                  stores_data[label_set] << [v, ts]
                 end
               ensure
                 store.close if store
@@ -181,30 +192,41 @@ module Prometheus
           end
 
           def aggregate_values(values)
-            if @values_aggregation_mode == SUM
-              values.inject { |sum, element| sum + element }
-            elsif @values_aggregation_mode == MAX
-              values.max
-            elsif @values_aggregation_mode == MIN
-              values.min
-            elsif @values_aggregation_mode == ALL
-              values.first
+            # Each entry in the `values` array is a tuple of `value` and `timestamp`,
+            # so for all aggregations except `MOST_RECENT`, we need to only take the
+            # first value in each entry and ignore the second.
+            if @values_aggregation_mode == MOST_RECENT
+              latest_tuple = values.max { |a,b| a[1] <=> b[1] }
+              latest_tuple.first # return the value without the timestamp
             else
-              raise InvalidStoreSettingsError,
-                    "Invalid Aggregation Mode: #{ @values_aggregation_mode }"
+              values = values.map(&:first) # Discard timestamps
+
+              if @values_aggregation_mode == SUM
+                values.inject { |sum, element| sum + element }
+              elsif @values_aggregation_mode == MAX
+                values.max
+              elsif @values_aggregation_mode == MIN
+                values.min
+              elsif @values_aggregation_mode == ALL
+                values.first
+              else
+                raise InvalidStoreSettingsError,
+                      "Invalid Aggregation Mode: #{ @values_aggregation_mode }"
+              end
             end
           end
         end
 
         private_constant :MetricStore
 
-        # A dict of doubles, backed by an file we access directly a a byte array.
+        # A dict of doubles, backed by an file we access directly as a byte array.
         #
         # The file starts with a 4 byte int, indicating how much of it is used.
         # Then 4 bytes of padding.
         # There's then a number of entries, consisting of a 4 byte int which is the
         # size of the next field, a utf-8 encoded string key, padding to an 8 byte
-        # alignment, and then a 8 byte float which is the value.
+        # alignment, and then a 8 byte float which is the value, and then a 8 byte
+        # float which is the unix timestamp when the value was set.
         class FileMappedDict
           INITIAL_FILE_SIZE = 1024*1024
 
@@ -235,8 +257,8 @@ module Prometheus
             with_file_lock do
               @positions.map do |key, pos|
                 @f.seek(pos)
-                value = @f.read(8).unpack('d')[0]
-                [key, value]
+                value, timestamp = @f.read(16).unpack('dd')
+                [key, value, timestamp]
               end
             end
           end
@@ -256,9 +278,25 @@ module Prometheus
               init_value(key)
             end
 
+            now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+            pos = @positions[key]
+            @f.seek(pos)
+            @f.write([value, now].pack('dd'))
+            @f.flush
+          end
+
+          def increment_value(key, by)
+            if !@positions.has_key?(key)
+              init_value(key)
+            end
+
             pos = @positions[key]
             @f.seek(pos)
-            @f.write([value].pack('d'))
+            value = @f.read(8).unpack('d')[0]
+
+            now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+            @f.seek(-8, :CUR)
+            @f.write([value + by, now].pack('dd'))
             @f.flush
           end
 
@@ -299,7 +337,7 @@ module Prometheus
           def init_value(key)
             # Pad to be 8-byte aligned.
             padded = key + (' ' * (8 - (key.length + 4) % 8))
-            value = [padded.length, padded, 0.0].pack("lA#{padded.length}d")
+            value = [padded.length, padded, 0.0, 0.0].pack("lA#{padded.length}dd")
             while @used + value.length > @capacity
               @capacity *= 2
               resize_file(@capacity)
@@ -310,7 +348,7 @@ module Prometheus
             @f.seek(0)
             @f.write([@used].pack('l'))
             @f.flush
-            @positions[key] = @used - 8
+            @positions[key] = @used - 16
           end
 
           # Read position of all keys. No locking is performed.
@@ -320,7 +358,7 @@ module Prometheus
               padded_len = @f.read(4).unpack('l')[0]
               key = @f.read(padded_len).unpack("A#{padded_len}")[0].strip
               @positions[key] = @f.pos
-              @f.seek(8, :CUR)
+              @f.seek(16, :CUR)
             end
           end
         end

data/lib/prometheus/client/histogram.rb

@@ -6,7 +6,7 @@ module Prometheus
   module Client
     # A histogram samples observations (usually things like request durations
     # or response sizes) and counts them in configurable buckets. It also
-    # provides a sum of all observed values.
+    # provides a total count and sum of all observed values.
     class Histogram < Metric
       # DEFAULT_BUCKETS are the default Histogram buckets. The default buckets
       # are tailored to broadly measure the response time (in seconds) of a
@@ -33,21 +33,41 @@ module Prometheus
               store_settings: store_settings)
       end
 
+      def self.linear_buckets(start:, width:, count:)
+        count.times.map { |idx| start.to_f + idx * width }
+      end
+
+      def self.exponential_buckets(start:, factor: 2, count:)
+        count.times.map { |idx| start.to_f * factor ** idx }
+      end
+
       def with_labels(labels)
-        self.class.new(name,
-                       docstring: docstring,
-                       labels: @labels,
-                       preset_labels: preset_labels.merge(labels),
-                       buckets: @buckets,
-                       store_settings: @store_settings)
+        new_metric = self.class.new(name,
+                                    docstring: docstring,
+                                    labels: @labels,
+                                    preset_labels: preset_labels.merge(labels),
+                                    buckets: @buckets,
+                                    store_settings: @store_settings)
+
+        # The new metric needs to use the same store as the "main" declared one, otherwise
+        # any observations on that copy with the pre-set labels won't actually be exported.
+        new_metric.replace_internal_store(@store)
+
+        new_metric
       end
 
       def type
         :histogram
       end
 
+      # Records a given value. The recorded value is usually positive
+      # or zero. A negative value is accepted but prevents current
+      # versions of Prometheus from properly detecting counter resets
+      # in the sum of observations. See
+      # https://prometheus.io/docs/practices/histograms/#count-and-sum-of-observations
+      # for details.
       def observe(value, labels: {})
-        bucket = buckets.find {|upper_limit| upper_limit > value  }
+        bucket = buckets.find {|upper_limit| upper_limit >= value  }
         bucket = "+Inf" if bucket.nil?
 
         base_label_set = label_set_for(labels)
@@ -81,19 +101,29 @@ module Prometheus
 
       # Returns all label sets with their values expressed as hashes with their buckets
       def values
-        v = @store.all_values
+        values = @store.all_values
 
-        result = v.each_with_object({}) do |(label_set, v), acc|
+        result = values.each_with_object({}) do |(label_set, v), acc|
           actual_label_set = label_set.reject{|l| l == :le }
           acc[actual_label_set] ||= @buckets.map{|b| [b.to_s, 0.0]}.to_h
           acc[actual_label_set][label_set[:le].to_s] = v
         end
 
-        result.each do |(label_set, v)|
+        result.each do |(_label_set, v)|
           accumulate_buckets(v)
         end
       end
 
+      def init_label_set(labels)
+        base_label_set = label_set_for(labels)
+
+        @store.synchronize do
+          (buckets + ["+Inf", "sum"]).each do |bucket|
+            @store.set(labels: base_label_set.merge(le: bucket.to_s), val: 0)
+          end
+        end
+      end
+
       private
 
       # Modifies the passed in parameter

data/lib/prometheus/client/label_set_validator.rb

@@ -5,8 +5,8 @@ module Prometheus
     # LabelSetValidator ensures that all used label sets comply with the
     # Prometheus specification.
     class LabelSetValidator
-      # TODO: we might allow setting :instance in the future
-      BASE_RESERVED_LABELS = [:job, :instance, :pid].freeze
+      BASE_RESERVED_LABELS = [:pid].freeze
+      LABEL_NAME_REGEX = /\A[a-zA-Z_][a-zA-Z0-9_]*\Z/
 
       class LabelSetError < StandardError; end
       class InvalidLabelSetError < LabelSetError; end
@@ -59,9 +59,15 @@ module Prometheus
       end
 
       def validate_name(key)
-        return true unless key.to_s.start_with?('__')
+        if key.to_s.start_with?('__')
+          raise ReservedLabelError, "label #{key} must not start with __"
+        end
+
+        unless key.to_s =~ LABEL_NAME_REGEX
+          raise InvalidLabelError, "label name must match /#{LABEL_NAME_REGEX}/"
+        end
 
-        raise ReservedLabelError, "label #{key} must not start with __"
+        true
       end
 
       def validate_reserved_key(key)

data/lib/prometheus/client/metric.rb

@@ -7,7 +7,7 @@ module Prometheus
   module Client
     # Metric
     class Metric
-      attr_reader :name, :docstring, :preset_labels
+      attr_reader :name, :docstring, :labels, :preset_labels
 
       def initialize(name,
                      docstring:,
@@ -29,18 +29,28 @@ module Prometheus
         @docstring = docstring
         @preset_labels = stringify_values(preset_labels)
 
+        @all_labels_preset = false
+        if preset_labels.keys.length == labels.length
+          @validator.validate_labelset!(preset_labels)
+          @all_labels_preset = true
+        end
+
         @store = Prometheus::Client.config.data_store.for_metric(
           name,
           metric_type: type,
           metric_settings: store_settings
         )
 
-        if preset_labels.keys.length == labels.length
-          @validator.validate_labelset!(preset_labels)
-          @all_labels_preset = true
-        end
+        # WARNING: Our internal store can be replaced later by `with_labels`
+        # Everything we do after this point needs to still work if @store gets replaced
+        init_label_set({}) if labels.empty?
+      end
+
+      protected def replace_internal_store(new_store)
+        @store = new_store
       end
 
+
       # Returns the value for the given label set
       def get(labels: {})
         label_set = label_set_for(labels)
@@ -48,11 +58,21 @@ module Prometheus
       end
 
       def with_labels(labels)
-        self.class.new(name,
-                       docstring: docstring,
-                       labels: @labels,
-                       preset_labels: preset_labels.merge(labels),
-                       store_settings: @store_settings)
+        new_metric = self.class.new(name,
+                                     docstring: docstring,
+                                     labels: @labels,
+                                     preset_labels: preset_labels.merge(labels),
+                                     store_settings: @store_settings)
+
+        # The new metric needs to use the same store as the "main" declared one, otherwise
+        # any observations on that copy with the pre-set labels won't actually be exported.
+        new_metric.replace_internal_store(@store)
+
+        new_metric
+      end
+
+      def init_label_set(labels)
+        @store.set(labels: label_set_for(labels), val: 0)
       end
 
       # Returns all label sets with their values

data/lib/prometheus/client/push.rb

@@ -1,11 +1,15 @@
 # encoding: UTF-8
 
+require 'base64'
 require 'thread'
 require 'net/http'
 require 'uri'
+require 'erb'
+require 'set'
 
 require 'prometheus/client'
 require 'prometheus/client/formats/text'
+require 'prometheus/client/label_set_validator'
 
 module Prometheus
   # Client is a ruby implementation for a Prometheus compatible client.
@@ -13,23 +17,41 @@ module Prometheus
     # Push implements a simple way to transmit a given registry to a given
     # Pushgateway.
     class Push
+      class HttpError < StandardError; end
+      class HttpRedirectError < HttpError; end
+      class HttpClientError < HttpError; end
+      class HttpServerError < HttpError; end
+
       DEFAULT_GATEWAY = 'http://localhost:9091'.freeze
       PATH            = '/metrics/job/%s'.freeze
-      INSTANCE_PATH   = '/metrics/job/%s/instance/%s'.freeze
       SUPPORTED_SCHEMES = %w(http https).freeze
 
-      attr_reader :job, :instance, :gateway, :path
+      attr_reader :job, :gateway, :path
+
+      def initialize(job:, gateway: DEFAULT_GATEWAY, grouping_key: {}, **kwargs)
+        raise ArgumentError, "job cannot be nil" if job.nil?
+        raise ArgumentError, "job cannot be empty" if job.empty?
+        @validator = LabelSetValidator.new(expected_labels: grouping_key.keys)
+        @validator.validate_symbols!(grouping_key)
 
-      def initialize(job, instance = nil, gateway = nil)
         @mutex = Mutex.new
         @job = job
-        @instance = instance
         @gateway = gateway || DEFAULT_GATEWAY
-        @path = build_path(job, instance)
+        @grouping_key = grouping_key
+        @path = build_path(job, grouping_key)
+
         @uri = parse("#{@gateway}#{@path}")
+        validate_no_basic_auth!(@uri)
 
         @http = Net::HTTP.new(@uri.host, @uri.port)
         @http.use_ssl = (@uri.scheme == 'https')
+        @http.open_timeout = kwargs[:open_timeout] if kwargs[:open_timeout]
+        @http.read_timeout = kwargs[:read_timeout] if kwargs[:read_timeout]
+      end
+
+      def basic_auth(user, password)
+        @user = user
+        @password = password
       end
 
       def add(registry)
@@ -64,26 +86,118 @@ module Prometheus
         raise ArgumentError, "#{url} is not a valid URL: #{e}"
       end
 
-      def build_path(job, instance)
-        if instance
-          format(INSTANCE_PATH, URI.escape(job), URI.escape(instance))
-        else
-          format(PATH, URI.escape(job))
+      def build_path(job, grouping_key)
+        path = format(PATH, ERB::Util::url_encode(job))
+
+        grouping_key.each do |label, value|
+          if value.include?('/')
+            encoded_value = Base64.urlsafe_encode64(value)
+            path += "/#{label}@base64/#{encoded_value}"
+          # While it's valid for the urlsafe_encode64 function to return an
+          # empty string when the input string is empty, it doesn't work for
+          # our specific use case as we're putting the result into a URL path
+          # segment. A double slash (`//`) can be normalised away by HTTP
+          # libraries, proxies, and web servers.
+          #
+          # For empty strings, we use a single padding character (`=`) as the
+          # value.
+          #
+          # See the pushgateway docs for more details:
+          #
+          # https://github.com/prometheus/pushgateway/blob/6393a901f56d4dda62cd0f6ab1f1f07c495b6354/README.md#url
+          elsif value.empty?
+            path += "/#{label}@base64/="
+          else
+            path += "/#{label}/#{ERB::Util::url_encode(value)}"
+          end
         end
+
+        path
       end
 
       def request(req_class, registry = nil)
+        validate_no_label_clashes!(registry) if registry
+
         req = req_class.new(@uri)
         req.content_type = Formats::Text::CONTENT_TYPE
-        req.basic_auth(@uri.user, @uri.password) if @uri.user
+        req.basic_auth(@user, @password) if @user
         req.body = Formats::Text.marshal(registry) if registry
 
-        @http.request(req)
+        response = @http.request(req)
+        validate_response!(response)
+
+        response
       end
 
       def synchronize
         @mutex.synchronize { yield }
       end
+
+      def validate_no_basic_auth!(uri)
+        if uri.user || uri.password
+          raise ArgumentError, <<~EOF
+            Setting Basic Auth credentials in the gateway URL is not supported, please call the `basic_auth` method.
+
+            Received username `#{uri.user}` in gateway URL. Instead of passing
+            Basic Auth credentials like this:
+
+            ```
+            push = Prometheus::Client::Push.new(job: "my-job", gateway: "http://user:password@localhost:9091")
+            ```
+
+            please pass them like this:
+
+            ```
+            push = Prometheus::Client::Push.new(job: "my-job", gateway: "http://localhost:9091")
+            push.basic_auth("user", "password")
+            ```
+
+            While URLs do support passing Basic Auth credentials using the
+            `http://user:password@example.com/` syntax, the username and
+            password in that syntax have to follow the usual rules for URL
+            encoding of characters per RFC 3986
+            (https://datatracker.ietf.org/doc/html/rfc3986#section-2.1).
+
+            Rather than place the burden of correctly performing that encoding
+            on users of this gem, we decided to have a separate method for
+            supplying Basic Auth credentials, with no requirement to URL encode
+            the characters in them.
+          EOF
+        end
+      end
+
+      def validate_no_label_clashes!(registry)
+        # There's nothing to check if we don't have a grouping key
+        return if @grouping_key.empty?
+
+        # We could be doing a lot of comparisons, so let's do them against a
+        # set rather than an array
+        grouping_key_labels = @grouping_key.keys.to_set
+
+        registry.metrics.each do |metric|
+          metric.labels.each do |label|
+            if grouping_key_labels.include?(label)
+              raise LabelSetValidator::InvalidLabelSetError,
+                "label :#{label} from grouping key collides with label of the " \
+                "same name from metric :#{metric.name} and would overwrite it"
+            end
+          end
+        end
+      end
+
+      def validate_response!(response)
+        status = Integer(response.code)
+        if status >= 300
+          message = "status: #{response.code}, message: #{response.message}, body: #{response.body}"
+          if status <= 399
+            raise HttpRedirectError, message
+          elsif status <= 499
+            raise HttpClientError, message
+          else
+            raise HttpServerError, message
+          end
+        end
+      end
     end
   end
 end

data/lib/prometheus/client/registry.rb

@@ -22,7 +22,7 @@ module Prometheus
         name = metric.name
 
         @mutex.synchronize do
-          if exist?(name.to_sym)
+          if @metrics.key?(name.to_sym)
             raise AlreadyRegisteredError, "#{name} has already been registered"
           end
           @metrics[name.to_sym] = metric
@@ -73,15 +73,15 @@ module Prometheus
       end
 
       def exist?(name)
-        @metrics.key?(name)
+        @mutex.synchronize { @metrics.key?(name) }
       end
 
       def get(name)
-        @metrics[name.to_sym]
+        @mutex.synchronize { @metrics[name.to_sym] }
       end
 
       def metrics
-        @metrics.values
+        @mutex.synchronize { @metrics.values }
       end
     end
   end

data/lib/prometheus/client/summary.rb

@@ -11,7 +11,12 @@ module Prometheus
         :summary
       end
 
-      # Records a given value.
+      # Records a given value. The recorded value is usually positive
+      # or zero. A negative value is accepted but prevents current
+      # versions of Prometheus from properly detecting counter resets
+      # in the sum of observations. See
+      # https://prometheus.io/docs/practices/histograms/#count-and-sum-of-observations
+      # for details.
       def observe(value, labels: {})
         base_label_set = label_set_for(labels)
 
@@ -36,15 +41,24 @@ module Prometheus
 
       # Returns all label sets with their values expressed as hashes with their sum/count
       def values
-        v = @store.all_values
+        values = @store.all_values
 
-        v.each_with_object({}) do |(label_set, v), acc|
+        values.each_with_object({}) do |(label_set, v), acc|
           actual_label_set = label_set.reject{|l| l == :quantile }
           acc[actual_label_set] ||= { "count" => 0.0, "sum" => 0.0 }
           acc[actual_label_set][label_set[:quantile]] = v
         end
       end
 
+      def init_label_set(labels)
+        base_label_set = label_set_for(labels)
+
+        @store.synchronize do
+          @store.set(labels: base_label_set.merge(quantile: "count"), val: 0)
+          @store.set(labels: base_label_set.merge(quantile: "sum"), val: 0)
+        end
+      end
+
       private
 
       def reserved_labels

data/lib/prometheus/client/version.rb

@@ -2,6 +2,6 @@
 
 module Prometheus
   module Client
-    VERSION = '1.0.0'
+    VERSION = '4.1.0'
   end
 end