prometheus-client 0.11.0.pre.alpha.1 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a558dc9a9c12b948a111369fd378ef1f41135695b0f7fb30eb2691a86a05ea3f
-  data.tar.gz: 0d8554c3e1a4cec8e6db066158247c7a1c8e0fc86653e9a252bc3c08c04d1618
+  metadata.gz: b7017e1e3c284f9558d758e91d4855ad97b5df63fdef115cee035aa0948aed30
+  data.tar.gz: 52a7abaadf1addf7bf9303d73a31a14338429882b488eb80c2144a6817cfdb7d
 SHA512:
-  metadata.gz: 91b4d4bb9f606b33f77bc0fab83a523d0cf61d6f875f5c42de738513aebc2e6a44994c9098c103b51b54eb2c80957c82616aab4fadb07e583c1b5b0ef3821697
-  data.tar.gz: d956b78c30fff9db52e45f9826f739f1088501ebd12ad71b016e0d58abe3262c320180fd68ba74dbc898a279141e6e27b9f51d2d8471b0d28f7e5bb5eef95595
+  metadata.gz: 1e3b2ff9368dacbdc175ea8cb709e1ad7be73b0d8e644468f65bf36f44b517798a28d7f5dab73b538519fc1a918f977e083e101986a761fdf2fe42ab719fcc1d
+  data.tar.gz: 7fb5a1887a88f7687f2af5ef4839e26030da25cc0c51a03445bbb43706caf8d2231c65108cf3265cc024bc286d5dc8a38ec5d29145b39b2f0cbd0277f29f1972

data/README.md

@@ -5,11 +5,17 @@ through a HTTP interface. Intended to be used together with a
 [Prometheus server][1].
 
 [![Gem Version][4]](http://badge.fury.io/rb/prometheus-client)
-[![Build Status][3]](http://travis-ci.org/prometheus/client_ruby)
-[![Coverage Status][7]](https://coveralls.io/r/prometheus/client_ruby)
+[![Build Status][3]](https://circleci.com/gh/prometheus/client_ruby/tree/master.svg?style=svg)
 
 ## Usage
 
+### Installation
+
+For a global installation run `gem install prometheus-client`.
+
+If you're using [Bundler](https://bundler.io/) add `gem "prometheus-client"` to your `Gemfile`.
+Make sure to run `bundle install` afterwards.
+
 ### Overview
 
 ```ruby
@@ -64,7 +70,7 @@ integrated [example application](examples/rack/README.md).
 The Ruby client can also be used to push its collected metrics to a
 [Pushgateway][8]. This comes in handy with batch jobs or in other scenarios
 where it's not possible or feasible to let a Prometheus server scrape a Ruby
-process. TLS and basic access authentication are supported.
+process. TLS and HTTP basic authentication are supported.
 
 ```ruby
 require 'prometheus/client'
@@ -74,18 +80,59 @@ registry = Prometheus::Client.registry
 # ... register some metrics, set/increment/observe/etc. their values
 
 # push the registry state to the default gateway
-Prometheus::Client::Push.new('my-batch-job').add(registry)
+Prometheus::Client::Push.new(job: 'my-batch-job').add(registry)
+
+# optional: specify a grouping key that uniquely identifies a job instance, and gateway.
+#
+# Note: the labels you use in the grouping key must not conflict with labels set on the
+# metrics being pushed. If they do, an error will be raised.
+Prometheus::Client::Push.new(
+  job: 'my-batch-job',
+  gateway: 'https://example.domain:1234',
+  grouping_key: { instance: 'some-instance', extra_key: 'foobar' }
+).add(registry)
+
+# If you want to replace any previously pushed metrics for a given grouping key,
+# use the #replace method.
+#
+# Unlike #add, this will completely replace the metrics under the specified grouping key
+# (i.e. anything currently present in the pushgateway for the specified grouping key, but
+# not present in the registry for that grouping key will be removed).
+#
+# See https://github.com/prometheus/pushgateway#put-method for a full explanation.
+Prometheus::Client::Push.new(job: 'my-batch-job').replace(registry)
+
+# If you want to delete all previously pushed metrics for a given grouping key,
+# use the #delete method.
+Prometheus::Client::Push.new(job: 'my-batch-job').delete
+```
 
-# optional: specify the instance name (instead of IP) and gateway.
-Prometheus::Client::Push.new('my-batch-job', 'foobar', 'https://example.domain:1234').add(registry)
+#### Basic authentication
 
-# If you want to replace any previously pushed metrics for a given instance,
-# use the #replace method.
-Prometheus::Client::Push.new('my-batch-job').replace(registry)
+By design, `Prometheus::Client::Push` doesn't read credentials for HTTP basic
+authentication when they are passed in via the gateway URL using the
+`http://user:password@example.com:9091` syntax, and will in fact raise an error if they're
+supplied that way.
 
-# If you want to delete all previously pushed metrics for a given instance,
-# use the #delete method.
-Prometheus::Client::Push.new('my-batch-job').delete
+The reason for this is that when using that syntax, the username and password
+have to follow the usual rules for URL encoding of characters [per RFC
+3986](https://datatracker.ietf.org/doc/html/rfc3986#section-2.1).
+
+Rather than place the burden of correctly performing that encoding on users of this gem,
+we decided to have a separate method for supplying HTTP basic authentication credentials,
+with no requirement to URL encode the characters in them.
+
+Instead of passing credentials like this:
+
+```ruby
+push = Prometheus::Client::Push.new(job: "my-job", gateway: "http://user:password@localhost:9091")
+```
+
+please pass them like this:
+
+```ruby
+push = Prometheus::Client::Push.new(job: "my-job", gateway: "http://localhost:9091")
+push.basic_auth("user", "password")
 ```
 
 ## Metrics
@@ -151,6 +198,11 @@ histogram.get(labels: { service: 'users' })
 # => { 0.005 => 3, 0.01 => 15, 0.025 => 18, ..., 2.5 => 42, 5 => 42, 10 = >42 }
 ```
 
+Histograms provide default buckets of `[0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]`
+
+You can specify your own buckets, either explicitly, or using the `Histogram.linear_buckets`
+or `Histogram.exponential_buckets` methods to define regularly spaced buckets.
+
 ### Summary
 
 Summary, similar to histograms, is an accumulator for samples. It captures
@@ -254,6 +306,12 @@ class MyComponent
 end
 ```
 
+### `init_label_set`
+
+The time series of a metric are not initialized until something happens. For counters, for example, this means that the time series do not exist until the counter is incremented for the first time.
+
+To get around this problem the client provides the `init_label_set` method that can be used to initialise the time series of a metric for a given label set.
+
 ### Reserved labels
 
 The following labels are reserved by the client library, and attempting to use them in a
@@ -271,7 +329,7 @@ is stored in a global Data Store object, rather than in the metric objects thems
 (This "storage" is ephemeral, generally in-memory, it's not "long-term storage")
 
 The main reason to do this is that different applications may have different requirements
-for their metrics storage. Application running in pre-fork servers (like Unicorn, for
+for their metrics storage. Applications running in pre-fork servers (like Unicorn, for
 example), require a shared store between all the processes, to be able to report coherent
 numbers. At the same time, other applications may not have this requirement but be very
 sensitive to performance, and would prefer instead a simpler, faster store.
@@ -307,11 +365,11 @@ When instantiating metrics, there is an optional `store_settings` attribute. Thi
 to set up store-specific settings for each metric. For most stores, this is not used, but
 for multi-process stores, this is used to specify how to aggregate the values of each
 metric across multiple processes. For the most part, this is used for Gauges, to specify
-whether you want to report the `SUM`, `MAX` or `MIN` value observed across all processes.
-For almost all other cases, you'd leave the default (`SUM`). More on this on the 
-*Aggregation* section below.
+whether you want to report the `SUM`, `MAX`, `MIN`, or `MOST_RECENT` value observed across
+all processes. For almost all other cases, you'd leave the default (`SUM`). More on this
+on the *Aggregation* section below.
 
-Other custom stores may also accept extra parameters besides `:aggregation`. See the
+Custom stores may also accept extra parameters besides `:aggregation`. See the
 documentation of each store for more details.
 
 ### Built-in stores
@@ -326,26 +384,73 @@ There are 3 built-in stores, with different trade-offs:
   it's absolutely not thread safe.
 - **DirectFileStore**: Stores data in binary files, one file per process and per metric.
   This is generally the recommended store to use with pre-fork servers and other 
-  "multi-process" scenarios.
-
-  Each metric gets a file for each process, and manages its contents by storing keys and
-  binary floats next to them, and updating the offsets of those Floats directly. When 
-  exporting metrics, it will find all the files that apply to each metric, read them, 
-  and aggregate them.
-
-  In order to do this, each Metric needs an `:aggregation` setting, specifying how
-  to aggregate the multiple possible values we can get for each labelset. By default,
-  they are `SUM`med, which is what most use-cases call for (counters and histograms,
-  for example). However, for Gauges, it's possible to set `MAX` or `MIN` as aggregation, 
-  to get the highest/lowest value of all the processes / threads.
-  
-  Even though this store saves data on disk, it's still much faster than would probably be 
-  expected, because the files are never actually `fsync`ed, so the store never blocks 
-  while waiting for disk. The kernel's page cache is incredibly efficient in this regard.
-  
-  If in doubt, check the benchmark scripts described in the documentation for creating 
-  your own stores and run them in your particular runtime environment to make sure this 
-  provides adequate performance.
+  "multi-process" scenarios. There are some important caveats to using this store, so
+  please read on the section below.
+
+### `DirectFileStore` caveats and things to keep in mind
+
+Each metric gets a file for each process, and manages its contents by storing keys and
+binary floats next to them, and updating the offsets of those Floats directly. When 
+exporting metrics, it will find all the files that apply to each metric, read them, 
+and aggregate them.
+
+**Aggregation of metrics**: Since there will be several files per metrics (one per process),
+these need to be aggregated to present a coherent view to Prometheus. Depending on your
+use case, you may need to control how this works. When using this store, 
+each Metric allows you to specify an `:aggregation` setting, defining how
+to aggregate the multiple possible values we can get for each labelset. By default,
+Counters, Histograms and Summaries are `SUM`med, and Gauges report all their values (one
+for each process), tagged with a `pid` label. You can also select `SUM`, `MAX`, `MIN`, or
+`MOST_RECENT` for your gauges, depending on your use case.
+
+Please note that that the `MOST_RECENT` aggregation only works for gauges, and it does not
+allow the use of `increment` / `decrement`, you can only use `set`. 
+
+**Memory Usage**: When scraped by Prometheus, this store will read all these files, get all
+the values and aggregate them. We have notice this can have a noticeable effect on memory
+usage for your app. We recommend you test this in a realistic usage scenario to make sure
+you won't hit any memory limits your app may have.
+
+**Resetting your metrics on each run**: You should also make sure that the directory where 
+you store your metric files (specified when initializing the `DirectFileStore`) is emptied 
+when your app starts. Otherwise, each app run will continue exporting the metrics from the 
+previous run.  
+
+If you have this issue, one way to do this is to run code similar to this as part of you
+initialization:
+
+```ruby
+Dir["#{app_path}/tmp/prometheus/*.bin"].each do |file_path|
+  File.unlink(file_path)
+end
+```
+
+If you are running in pre-fork servers (such as Unicorn or Puma with multiple processes),
+make sure you do this **before** the server forks. Otherwise, each child process may delete
+files created by other processes on *this* run, instead of deleting old files.
+
+**Declare metrics before fork**: As well as deleting files before your process forks, you
+should make sure to declare your metrics before forking too. Because the metric registry
+is held in memory, any metrics declared after forking will only be present in child
+processes where the code declaring them ran, and as a result may not be consistently
+exported when scraped (i.e. they will only appear when a child process that declared them
+is scraped).
+
+If you're absolutely sure that every child process will run the metric declaration code,
+then you won't run into this issue, but the simplest approach is to declare the metrics
+before forking.
+
+**Large numbers of files**: Because there is an individual file per metric and per process 
+(which is done to optimize for observation performance), you may end up with a large number 
+of files. We don't currently have a solution for this problem, but we're working on it.
+
+**Performance**: Even though this store saves data on disk, it's still much faster than 
+would probably be expected, because the files are never actually `fsync`ed, so the store 
+never blocks while waiting for disk. The kernel's page cache is incredibly efficient in 
+this regard. If in doubt, check the benchmark scripts described in the documentation for 
+creating your own stores and run them in your particular runtime environment to make sure 
+this provides adequate performance.
+
 
 ### Building your own store, and stores other than the built-in ones.
 
@@ -364,16 +469,16 @@ If you are in a multi-process environment (such as pre-fork servers like Unicorn
 process will probably keep their own counters, which need to be aggregated when receiving
 a Prometheus scrape, to report coherent total numbers.
 
-For Counters and Histograms (and quantile-less Summaries), this is simply a matter of 
+For Counters, Histograms and quantile-less Summaries this is simply a matter of 
 summing the values of each process.
 
 For Gauges, however, this may not be the right thing to do, depending on what they're 
 measuring. You might want to take the maximum or minimum value observed in any process,
-rather than the sum of all of them. You may also want to export each process's individual
-value.
+rather than the sum of all of them. By default, we export each process's individual
+value, with a `pid` label identifying each one.
 
-In those cases, you should use the `store_settings` parameter when registering the 
-metric, to specify an `:aggregation` setting. 
+If these defaults don't work for your use case, you should use the `store_settings` 
+parameter when registering the metric, to specify an `:aggregation` setting. 
 
 ```ruby
 free_disk_space = registry.gauge(:free_disk_space_bytes,

data/lib/prometheus/client/data_stores/README.md

@@ -187,7 +187,7 @@ has created a good amount of research, benchmarks, and experimental stores, whic
 weren't useful to include in this repo, but may be a useful resource or starting point 
 if you are building your own store.
 
-Check out the [GoCardless Data Stores Experiments](gocardless/prometheus-client-ruby-data-stores-experiments) 
+Check out the [GoCardless Data Stores Experiments](https://github.com/gocardless/prometheus-client-ruby-data-stores-experiments) 
 repository for these.
 
 ## Sample, imaginary multi-process Data Store

data/lib/prometheus/client/data_stores/direct_file_store.rb

@@ -18,14 +18,18 @@ module Prometheus
       #
       # In order to do this, each Metric needs an `:aggregation` setting, specifying how
       # to aggregate the multiple possible values we can get for each labelset. By default,
-      # they are `SUM`med, which is what most use cases call for (counters and histograms,
-      # for example).
-      # However, for Gauges, it's possible to set `MAX` or `MIN` as aggregation, to get
-      # the highest value of all the processes / threads.
+      # Counters, Histograms and Summaries get `SUM`med, and Gauges will report `ALL`
+      # values, tagging each one with a `pid` label.
+      # For Gauges, it's also possible to set `SUM`, MAX` or `MIN` as aggregation, to get
+      # the highest / lowest value / or the sum of all the processes / threads.
+      #
+      # Before using this Store, please read the "`DirectFileStore` caveats and things to
+      # keep in mind" section of the main README in this repository. It includes a number
+      # of important things to keep in mind.
 
       class DirectFileStore
         class InvalidStoreSettingsError < StandardError; end
-        AGGREGATION_MODES = [MAX = :max, MIN = :min, SUM = :sum, ALL = :all]
+        AGGREGATION_MODES = [MAX = :max, MIN = :min, SUM = :sum, ALL = :all, MOST_RECENT = :most_recent]
         DEFAULT_METRIC_SETTINGS = { aggregation: SUM }
         DEFAULT_GAUGE_SETTINGS = { aggregation: ALL }
 
@@ -41,7 +45,7 @@ module Prometheus
           end
 
           settings = default_settings.merge(metric_settings)
-          validate_metric_settings(settings)
+          validate_metric_settings(metric_type, settings)
 
           MetricStore.new(metric_name: metric_name,
                           store_settings: @store_settings,
@@ -50,7 +54,7 @@ module Prometheus
 
         private
 
-        def validate_metric_settings(metric_settings)
+        def validate_metric_settings(metric_type, metric_settings)
           unless metric_settings.has_key?(:aggregation) &&
             AGGREGATION_MODES.include?(metric_settings[:aggregation])
             raise InvalidStoreSettingsError,
@@ -61,6 +65,11 @@ module Prometheus
             raise InvalidStoreSettingsError,
                   "Only :aggregation setting can be specified"
           end
+
+          if metric_settings[:aggregation] == MOST_RECENT && metric_type != :gauge
+            raise InvalidStoreSettingsError,
+                  "Only :gauge metrics support :most_recent aggregation"
+          end
         end
 
         class MetricStore
@@ -70,6 +79,7 @@ module Prometheus
             @metric_name = metric_name
             @store_settings = store_settings
             @values_aggregation_mode = metric_settings[:aggregation]
+            @store_opened_by_pid = nil
 
             @lock = Monitor.new
           end
@@ -96,6 +106,12 @@ module Prometheus
           end
 
           def increment(labels:, by: 1)
+            if @values_aggregation_mode == DirectFileStore::MOST_RECENT
+              raise InvalidStoreSettingsError,
+                    "The :most_recent aggregation does not support the use of increment"\
+                      "/decrement"
+            end
+
             key = store_key(labels)
             in_process_sync do
               value = internal_store.read_value(key)
@@ -117,7 +133,7 @@ module Prometheus
             stores_for_metric.each do |file_path|
               begin
                 store = FileMappedDict.new(file_path, true)
-                store.all_values.each do |(labelset_qs, v)|
+                store.all_values.each do |(labelset_qs, v, ts)|
                   # Labels come as a query string, and CGI::parse returns arrays for each key
                   # "foo=bar&x=y" => { "foo" => ["bar"], "x" => ["y"] }
                   # Turn the keys back into symbols, and remove the arrays
@@ -125,7 +141,7 @@ module Prometheus
                     [k.to_sym, vs.first]
                   end.to_h
 
-                  stores_data[label_set] << v
+                  stores_data[label_set] << [v, ts]
                 end
               ensure
                 store.close if store
@@ -177,30 +193,41 @@ module Prometheus
           end
 
           def aggregate_values(values)
-            if @values_aggregation_mode == SUM
-              values.inject { |sum, element| sum + element }
-            elsif @values_aggregation_mode == MAX
-              values.max
-            elsif @values_aggregation_mode == MIN
-              values.min
-            elsif @values_aggregation_mode == ALL
-              values.first
+            # Each entry in the `values` array is a tuple of `value` and `timestamp`,
+            # so for all aggregations except `MOST_RECENT`, we need to only take the
+            # first value in each entry and ignore the second.
+            if @values_aggregation_mode == MOST_RECENT
+              latest_tuple = values.max { |a,b| a[1] <=> b[1] }
+              latest_tuple.first # return the value without the timestamp
             else
-              raise InvalidStoreSettingsError,
-                    "Invalid Aggregation Mode: #{ @values_aggregation_mode }"
+              values = values.map(&:first) # Discard timestamps
+
+              if @values_aggregation_mode == SUM
+                values.inject { |sum, element| sum + element }
+              elsif @values_aggregation_mode == MAX
+                values.max
+              elsif @values_aggregation_mode == MIN
+                values.min
+              elsif @values_aggregation_mode == ALL
+                values.first
+              else
+                raise InvalidStoreSettingsError,
+                      "Invalid Aggregation Mode: #{ @values_aggregation_mode }"
+              end
             end
           end
         end
 
         private_constant :MetricStore
 
-        # A dict of doubles, backed by an file we access directly a a byte array.
+        # A dict of doubles, backed by an file we access directly as a byte array.
         #
         # The file starts with a 4 byte int, indicating how much of it is used.
         # Then 4 bytes of padding.
         # There's then a number of entries, consisting of a 4 byte int which is the
         # size of the next field, a utf-8 encoded string key, padding to an 8 byte
-        # alignment, and then a 8 byte float which is the value.
+        # alignment, and then a 8 byte float which is the value, and then a 8 byte
+        # float which is the unix timestamp when the value was set.
         class FileMappedDict
           INITIAL_FILE_SIZE = 1024*1024
 
@@ -231,8 +258,8 @@ module Prometheus
             with_file_lock do
               @positions.map do |key, pos|
                 @f.seek(pos)
-                value = @f.read(8).unpack('d')[0]
-                [key, value]
+                value, timestamp = @f.read(16).unpack('dd')
+                [key, value, timestamp]
               end
             end
           end
@@ -252,9 +279,10 @@ module Prometheus
               init_value(key)
             end
 
+            now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
             pos = @positions[key]
             @f.seek(pos)
-            @f.write([value].pack('d'))
+            @f.write([value, now].pack('dd'))
             @f.flush
           end
 
@@ -295,7 +323,7 @@ module Prometheus
           def init_value(key)
             # Pad to be 8-byte aligned.
             padded = key + (' ' * (8 - (key.length + 4) % 8))
-            value = [padded.length, padded, 0.0].pack("lA#{padded.length}d")
+            value = [padded.length, padded, 0.0, 0.0].pack("lA#{padded.length}dd")
             while @used + value.length > @capacity
               @capacity *= 2
               resize_file(@capacity)
@@ -306,7 +334,7 @@ module Prometheus
             @f.seek(0)
             @f.write([@used].pack('l'))
             @f.flush
-            @positions[key] = @used - 8
+            @positions[key] = @used - 16
           end
 
           # Read position of all keys. No locking is performed.
@@ -316,7 +344,7 @@ module Prometheus
               padded_len = @f.read(4).unpack('l')[0]
               key = @f.read(padded_len).unpack("A#{padded_len}")[0].strip
               @positions[key] = @f.pos
-              @f.seek(8, :CUR)
+              @f.seek(16, :CUR)
             end
           end
         end

data/lib/prometheus/client/histogram.rb

@@ -6,7 +6,7 @@ module Prometheus
   module Client
     # A histogram samples observations (usually things like request durations
     # or response sizes) and counts them in configurable buckets. It also
-    # provides a sum of all observed values.
+    # provides a total count and sum of all observed values.
     class Histogram < Metric
       # DEFAULT_BUCKETS are the default Histogram buckets. The default buckets
       # are tailored to broadly measure the response time (in seconds) of a
@@ -33,21 +33,41 @@ module Prometheus
               store_settings: store_settings)
       end
 
+      def self.linear_buckets(start:, width:, count:)
+        count.times.map { |idx| start.to_f + idx * width }
+      end
+
+      def self.exponential_buckets(start:, factor: 2, count:)
+        count.times.map { |idx| start.to_f * factor ** idx }
+      end
+
       def with_labels(labels)
-        self.class.new(name,
-                       docstring: docstring,
-                       labels: @labels,
-                       preset_labels: preset_labels.merge(labels),
-                       buckets: @buckets,
-                       store_settings: @store_settings)
+        new_metric = self.class.new(name,
+                                    docstring: docstring,
+                                    labels: @labels,
+                                    preset_labels: preset_labels.merge(labels),
+                                    buckets: @buckets,
+                                    store_settings: @store_settings)
+
+        # The new metric needs to use the same store as the "main" declared one, otherwise
+        # any observations on that copy with the pre-set labels won't actually be exported.
+        new_metric.replace_internal_store(@store)
+
+        new_metric
       end
 
       def type
         :histogram
       end
 
+      # Records a given value. The recorded value is usually positive
+      # or zero. A negative value is accepted but prevents current
+      # versions of Prometheus from properly detecting counter resets
+      # in the sum of observations. See
+      # https://prometheus.io/docs/practices/histograms/#count-and-sum-of-observations
+      # for details.
       def observe(value, labels: {})
-        bucket = buckets.find {|upper_limit| upper_limit > value  }
+        bucket = buckets.find {|upper_limit| upper_limit >= value  }
         bucket = "+Inf" if bucket.nil?
 
         base_label_set = label_set_for(labels)
@@ -81,19 +101,29 @@ module Prometheus
 
       # Returns all label sets with their values expressed as hashes with their buckets
       def values
-        v = @store.all_values
+        values = @store.all_values
 
-        result = v.each_with_object({}) do |(label_set, v), acc|
+        result = values.each_with_object({}) do |(label_set, v), acc|
           actual_label_set = label_set.reject{|l| l == :le }
           acc[actual_label_set] ||= @buckets.map{|b| [b.to_s, 0.0]}.to_h
           acc[actual_label_set][label_set[:le].to_s] = v
         end
 
-        result.each do |(label_set, v)|
+        result.each do |(_label_set, v)|
           accumulate_buckets(v)
         end
       end
 
+      def init_label_set(labels)
+        base_label_set = label_set_for(labels)
+
+        @store.synchronize do
+          (buckets + ["+Inf", "sum"]).each do |bucket|
+            @store.set(labels: base_label_set.merge(le: bucket.to_s), val: 0)
+          end
+        end
+      end
+
       private
 
       # Modifies the passed in parameter

data/lib/prometheus/client/label_set_validator.rb

@@ -7,6 +7,7 @@ module Prometheus
     class LabelSetValidator
       # TODO: we might allow setting :instance in the future
       BASE_RESERVED_LABELS = [:job, :instance, :pid].freeze
+      LABEL_NAME_REGEX = /\A[a-zA-Z_][a-zA-Z0-9_]*\Z/
 
       class LabelSetError < StandardError; end
       class InvalidLabelSetError < LabelSetError; end
@@ -59,9 +60,15 @@ module Prometheus
       end
 
       def validate_name(key)
-        return true unless key.to_s.start_with?('__')
+        if key.to_s.start_with?('__')
+          raise ReservedLabelError, "label #{key} must not start with __"
+        end
+
+        unless key.to_s =~ LABEL_NAME_REGEX
+          raise InvalidLabelError, "label name must match /#{LABEL_NAME_REGEX}/"
+        end
 
-        raise ReservedLabelError, "label #{key} must not start with __"
+        true
       end
 
       def validate_reserved_key(key)

data/lib/prometheus/client/metric.rb

@@ -7,7 +7,7 @@ module Prometheus
   module Client
     # Metric
     class Metric
-      attr_reader :name, :docstring, :preset_labels
+      attr_reader :name, :docstring, :labels, :preset_labels
 
       def initialize(name,
                      docstring:,
@@ -29,18 +29,28 @@ module Prometheus
         @docstring = docstring
         @preset_labels = stringify_values(preset_labels)
 
+        @all_labels_preset = false
+        if preset_labels.keys.length == labels.length
+          @validator.validate_labelset!(preset_labels)
+          @all_labels_preset = true
+        end
+
         @store = Prometheus::Client.config.data_store.for_metric(
           name,
           metric_type: type,
           metric_settings: store_settings
         )
 
-        if preset_labels.keys.length == labels.length
-          @validator.validate_labelset!(preset_labels)
-          @all_labels_preset = true
-        end
+        # WARNING: Our internal store can be replaced later by `with_labels`
+        # Everything we do after this point needs to still work if @store gets replaced
+        init_label_set({}) if labels.empty?
+      end
+
+      protected def replace_internal_store(new_store)
+        @store = new_store
       end
 
+
       # Returns the value for the given label set
       def get(labels: {})
         label_set = label_set_for(labels)
@@ -48,11 +58,21 @@ module Prometheus
       end
 
       def with_labels(labels)
-        self.class.new(name,
-                       docstring: docstring,
-                       labels: @labels,
-                       preset_labels: preset_labels.merge(labels),
-                       store_settings: @store_settings)
+        new_metric = self.class.new(name,
+                                     docstring: docstring,
+                                     labels: @labels,
+                                     preset_labels: preset_labels.merge(labels),
+                                     store_settings: @store_settings)
+
+        # The new metric needs to use the same store as the "main" declared one, otherwise
+        # any observations on that copy with the pre-set labels won't actually be exported.
+        new_metric.replace_internal_store(@store)
+
+        new_metric
+      end
+
+      def init_label_set(labels)
+        @store.set(labels: label_set_for(labels), val: 0)
       end
 
       # Returns all label sets with their values

data/lib/prometheus/client/push.rb

@@ -1,11 +1,15 @@
 # encoding: UTF-8
 
+require 'base64'
 require 'thread'
 require 'net/http'
 require 'uri'
+require 'erb'
+require 'set'
 
 require 'prometheus/client'
 require 'prometheus/client/formats/text'
+require 'prometheus/client/label_set_validator'
 
 module Prometheus
   # Client is a ruby implementation for a Prometheus compatible client.
@@ -13,23 +17,41 @@ module Prometheus
     # Push implements a simple way to transmit a given registry to a given
     # Pushgateway.
     class Push
+      class HttpError < StandardError; end
+      class HttpRedirectError < HttpError; end
+      class HttpClientError < HttpError; end
+      class HttpServerError < HttpError; end
+
       DEFAULT_GATEWAY = 'http://localhost:9091'.freeze
       PATH            = '/metrics/job/%s'.freeze
-      INSTANCE_PATH   = '/metrics/job/%s/instance/%s'.freeze
       SUPPORTED_SCHEMES = %w(http https).freeze
 
-      attr_reader :job, :instance, :gateway, :path
+      attr_reader :job, :gateway, :path
+
+      def initialize(job:, gateway: DEFAULT_GATEWAY, grouping_key: {}, **kwargs)
+        raise ArgumentError, "job cannot be nil" if job.nil?
+        raise ArgumentError, "job cannot be empty" if job.empty?
+        @validator = LabelSetValidator.new(expected_labels: grouping_key.keys)
+        @validator.validate_symbols!(grouping_key)
 
-      def initialize(job, instance = nil, gateway = nil)
         @mutex = Mutex.new
         @job = job
-        @instance = instance
         @gateway = gateway || DEFAULT_GATEWAY
-        @path = build_path(job, instance)
+        @grouping_key = grouping_key
+        @path = build_path(job, grouping_key)
+
         @uri = parse("#{@gateway}#{@path}")
+        validate_no_basic_auth!(@uri)
 
         @http = Net::HTTP.new(@uri.host, @uri.port)
         @http.use_ssl = (@uri.scheme == 'https')
+        @http.open_timeout = kwargs[:open_timeout] if kwargs[:open_timeout]
+        @http.read_timeout = kwargs[:read_timeout] if kwargs[:read_timeout]
+      end
+
+      def basic_auth(user, password)
+        @user = user
+        @password = password
       end
 
       def add(registry)
@@ -64,26 +86,118 @@ module Prometheus
         raise ArgumentError, "#{url} is not a valid URL: #{e}"
       end
 
-      def build_path(job, instance)
-        if instance
-          format(INSTANCE_PATH, URI.escape(job), URI.escape(instance))
-        else
-          format(PATH, URI.escape(job))
+      def build_path(job, grouping_key)
+        path = format(PATH, ERB::Util::url_encode(job))
+
+        grouping_key.each do |label, value|
+          if value.include?('/')
+            encoded_value = Base64.urlsafe_encode64(value)
+            path += "/#{label}@base64/#{encoded_value}"
+          # While it's valid for the urlsafe_encode64 function to return an
+          # empty string when the input string is empty, it doesn't work for
+          # our specific use case as we're putting the result into a URL path
+          # segment. A double slash (`//`) can be normalised away by HTTP
+          # libraries, proxies, and web servers.
+          #
+          # For empty strings, we use a single padding character (`=`) as the
+          # value.
+          #
+          # See the pushgateway docs for more details:
+          #
+          # https://github.com/prometheus/pushgateway/blob/6393a901f56d4dda62cd0f6ab1f1f07c495b6354/README.md#url
+          elsif value.empty?
+            path += "/#{label}@base64/="
+          else
+            path += "/#{label}/#{ERB::Util::url_encode(value)}"
+          end
         end
+
+        path
       end
 
       def request(req_class, registry = nil)
+        validate_no_label_clashes!(registry) if registry
+
         req = req_class.new(@uri)
         req.content_type = Formats::Text::CONTENT_TYPE
-        req.basic_auth(@uri.user, @uri.password) if @uri.user
+        req.basic_auth(@user, @password) if @user
         req.body = Formats::Text.marshal(registry) if registry
 
-        @http.request(req)
+        response = @http.request(req)
+        validate_response!(response)
+
+        response
       end
 
       def synchronize
         @mutex.synchronize { yield }
       end
+
+      def validate_no_basic_auth!(uri)
+        if uri.user || uri.password
+          raise ArgumentError, <<~EOF
+            Setting Basic Auth credentials in the gateway URL is not supported, please call the `basic_auth` method.
+
+            Received username `#{uri.user}` in gateway URL. Instead of passing
+            Basic Auth credentials like this:
+
+            ```
+            push = Prometheus::Client::Push.new(job: "my-job", gateway: "http://user:password@localhost:9091")
+            ```
+
+            please pass them like this:
+
+            ```
+            push = Prometheus::Client::Push.new(job: "my-job", gateway: "http://localhost:9091")
+            push.basic_auth("user", "password")
+            ```
+
+            While URLs do support passing Basic Auth credentials using the
+            `http://user:password@example.com/` syntax, the username and
+            password in that syntax have to follow the usual rules for URL
+            encoding of characters per RFC 3986
+            (https://datatracker.ietf.org/doc/html/rfc3986#section-2.1).
+
+            Rather than place the burden of correctly performing that encoding
+            on users of this gem, we decided to have a separate method for
+            supplying Basic Auth credentials, with no requirement to URL encode
+            the characters in them.
+          EOF
+        end
+      end
+
+      def validate_no_label_clashes!(registry)
+        # There's nothing to check if we don't have a grouping key
+        return if @grouping_key.empty?
+
+        # We could be doing a lot of comparisons, so let's do them against a
+        # set rather than an array
+        grouping_key_labels = @grouping_key.keys.to_set
+
+        registry.metrics.each do |metric|
+          metric.labels.each do |label|
+            if grouping_key_labels.include?(label)
+              raise LabelSetValidator::InvalidLabelSetError,
+                "label :#{label} from grouping key collides with label of the " \
+                "same name from metric :#{metric.name} and would overwrite it"
+            end
+          end
+        end
+      end
+
+      def validate_response!(response)
+        status = Integer(response.code)
+        if status >= 300
+          message = "status: #{response.code}, message: #{response.message}, body: #{response.body}"
+          if status <= 399
+            raise HttpRedirectError, message
+          elsif status <= 499
+            raise HttpClientError, message
+          else
+            raise HttpServerError, message
+          end
+        end
+      end
     end
   end
 end

data/lib/prometheus/client/registry.rb

@@ -22,7 +22,7 @@ module Prometheus
         name = metric.name
 
         @mutex.synchronize do
-          if exist?(name.to_sym)
+          if @metrics.key?(name.to_sym)
             raise AlreadyRegisteredError, "#{name} has already been registered"
           end
           @metrics[name.to_sym] = metric
@@ -73,15 +73,15 @@ module Prometheus
       end
 
       def exist?(name)
-        @metrics.key?(name)
+        @mutex.synchronize { @metrics.key?(name) }
       end
 
       def get(name)
-        @metrics[name.to_sym]
+        @mutex.synchronize { @metrics[name.to_sym] }
       end
 
       def metrics
-        @metrics.values
+        @mutex.synchronize { @metrics.values }
       end
     end
   end

data/lib/prometheus/client/summary.rb

@@ -11,7 +11,12 @@ module Prometheus
         :summary
       end
 
-      # Records a given value.
+      # Records a given value. The recorded value is usually positive
+      # or zero. A negative value is accepted but prevents current
+      # versions of Prometheus from properly detecting counter resets
+      # in the sum of observations. See
+      # https://prometheus.io/docs/practices/histograms/#count-and-sum-of-observations
+      # for details.
       def observe(value, labels: {})
         base_label_set = label_set_for(labels)
 
@@ -36,15 +41,24 @@ module Prometheus
 
       # Returns all label sets with their values expressed as hashes with their sum/count
       def values
-        v = @store.all_values
+        values = @store.all_values
 
-        v.each_with_object({}) do |(label_set, v), acc|
+        values.each_with_object({}) do |(label_set, v), acc|
           actual_label_set = label_set.reject{|l| l == :quantile }
           acc[actual_label_set] ||= { "count" => 0.0, "sum" => 0.0 }
           acc[actual_label_set][label_set[:quantile]] = v
         end
       end
 
+      def init_label_set(labels)
+        base_label_set = label_set_for(labels)
+
+        @store.synchronize do
+          @store.set(labels: base_label_set.merge(quantile: "count"), val: 0)
+          @store.set(labels: base_label_set.merge(quantile: "sum"), val: 0)
+        end
+      end
+
       private
 
       def reserved_labels

data/lib/prometheus/client/version.rb

@@ -2,6 +2,6 @@
 
 module Prometheus
   module Client
-    VERSION = '0.11.0-alpha.1'
+    VERSION = '3.0.0'
   end
 end

data/lib/prometheus/middleware/collector.rb

@@ -67,15 +67,17 @@ module Prometheus
       end
 
       def record(env, code, duration)
+        path = generate_path(env)
+
         counter_labels = {
           code:   code,
           method: env['REQUEST_METHOD'].downcase,
-          path:   strip_ids_from_path(env['PATH_INFO']),
+          path:   strip_ids_from_path(path),
         }
 
         duration_labels = {
           method: env['REQUEST_METHOD'].downcase,
-          path:   strip_ids_from_path(env['PATH_INFO']),
+          path:   strip_ids_from_path(path),
         }
 
         @requests.increment(labels: counter_labels)
@@ -85,10 +87,58 @@ module Prometheus
         nil
       end
 
+      # While `PATH_INFO` is framework agnostic, and works for any Rack app, some Ruby web
+      # frameworks pass a more useful piece of information into the request env - the
+      # route that the request matched.
+      #
+      # This means that rather than using our generic `:id` and `:uuid` replacements in
+      # the `path` label for any path segments that look like dynamic IDs, we can put the
+      # actual route that matched in there, with correctly named parameters. For example,
+      # if a Sinatra app defined a route like:
+      #
+      # get "/foo/:bar" do
+      #   ...
+      # end
+      #
+      # instead of containing `/foo/:id`, the `path` label would contain `/foo/:bar`.
+      #
+      # Sadly, Rails is a notable exception, and (as far as I can tell at the time of
+      # writing) doesn't provide this info in the request env.
+      def generate_path(env)
+        if env['sinatra.route']
+          route = env['sinatra.route'].partition(' ').last
+        elsif env['grape.routing_args']
+          # We are deep in the weeds of an object that Grape passes into the request env,
+          # but don't document any explicit guarantees about. Let's have a fallback in
+          # case they change it down the line.
+          #
+          # This code would be neater with the safe navigation operator (`&.`) here rather
+          # than the much more verbose `respond_to?` calls, but unlike Rails' `try`
+          # method, it still raises an error if the object is non-nil, but doesn't respond
+          # to the method being called on it.
+          route = nil
+
+          route_info = env.dig('grape.routing_args', :route_info)
+          if route_info.respond_to?(:pattern)
+            pattern = route_info.pattern
+            if pattern.respond_to?(:origin)
+              route = pattern.origin
+            end
+          end
+
+          # Fall back to PATH_INFO if Grape change the structure of `grape.routing_args`
+          route ||= env['PATH_INFO']
+        else
+          route = env['PATH_INFO']
+        end
+
+        [env['SCRIPT_NAME'], route].join
+      end
+
       def strip_ids_from_path(path)
         path
-          .gsub(%r{/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}(/|$)}, '/:uuid\\1')
-          .gsub(%r{/\d+(/|$)}, '/:id\\1')
+          .gsub(%r{/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}(?=/|$)}, '/:uuid\\1')
+          .gsub(%r{/\d+(?=/|$)}, '/:id\\1')
       end
     end
   end

data/lib/prometheus/middleware/exporter.rb

@@ -21,11 +21,12 @@ module Prometheus
         @app = app
         @registry = options[:registry] || Client.registry
         @path = options[:path] || '/metrics'
+        @port = options[:port]
         @acceptable = build_dictionary(FORMATS, FALLBACK)
       end
 
       def call(env)
-        if env['PATH_INFO'] == @path
+        if metrics_port?(env['SERVER_PORT']) && env['PATH_INFO'] == @path
           format = negotiate(env, @acceptable)
           format ? respond_with(format) : not_acceptable(FORMATS)
         else
@@ -86,6 +87,10 @@ module Prometheus
           memo[format::MEDIA_TYPE] = format
         end
       end
+
+      def metrics_port?(request_port)
+        @port.nil? || @port.to_s == request_port
+      end
     end
   end
 end

metadata

@@ -1,16 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: prometheus-client
 version: !ruby/object:Gem::Version
-  version: 0.11.0.pre.alpha.1
+  version: 3.0.0
 platform: ruby
 authors:
 - Ben Kochie
 - Chris Sinjakli
 - Daniel Magliola
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-10-28 00:00:00.000000000 Z
+date: 2022-02-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: benchmark-ips
@@ -40,10 +40,10 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: 
+description:
 email:
 - superq@gmail.com
-- chris@gocardless.com
+- chris@sinjakli.co.uk
 - dmagliola@crystalgears.com
 executables: []
 extensions: []
@@ -73,7 +73,7 @@ homepage: https://github.com/prometheus/client_ruby
 licenses:
 - Apache 2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -84,13 +84,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
-signing_key: 
+rubygems_version: 3.2.32
+signing_key:
 specification_version: 4
 summary: A suite of instrumentation metric primitivesthat can be exposed through a
   web services interface.