prometheus_exporter 1.0.1 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9a885aa655d0bd9d939bef77ec5d7c37027d19882a96b184d0351c49593393b9
-  data.tar.gz: bca2080108cb8169d5589ee481fc40b0edb39e890a8a0fba6ec301a987c35396
+  metadata.gz: f5e708b8c63be6ecff9750deebf18cad149d9b4fa7350a4523af2f145caffec5
+  data.tar.gz: 1204a53af367dc60e24ded2e1630e2b447ef79d3caff32165ac5c40b62955ded
 SHA512:
-  metadata.gz: b21fe8ad31aec32b877ab50146759715c41968dd1edc510af7b986436983217fd7eb0c468faf2ceca7897ae855a36a88f3b6aef1e0dcf776846051e4cd35b822
-  data.tar.gz: 5a0a61bd914c269385bb6b46df97b2eca5e6fe4573092f10dfed7d92a4f052b62761faa53902bfdf93e2286dc96a13bc804dd56e820dec99c95a00550026b741
+  metadata.gz: 5dee50712e73be92702e2948a0a69aa8b8007df59de35f10c5980d9d5a2653092a4247e7ad207713460df80f59bb0691835a3aa2b04fa751d28cc5b9c3f66f48
+  data.tar.gz: 0af7415fd207b1eb5f7f62639f8d48bc7735df9397537524c536631d24150b9b7daa87d17652f070a79f8134d50e780c9143ddeedcfc2d44eaa27586ffe64b19

data/.github/workflows/ci.yml

@@ -6,7 +6,7 @@ on:
       - main
   pull_request:
   schedule:
-    - cron: '0 0 * * 0' # weekly
+    - cron: "0 0 * * 0" # weekly
 
 jobs:
   build:
@@ -20,7 +20,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [2.6, 2.7, 3.0]
+        ruby: ['2.6', '2.7', '3.0', '3.1']
         activerecord: [60, 61]
 
     steps:

data/CHANGELOG

@@ -1,3 +1,10 @@
+2.0.0 - 2022-02-18
+
+- FEATURE: Add per worker custom labels 
+- FEATURE: support custom histogram buckets
+- FIX: all metrics are exposing status label, and not only `http_requests_total`
+- BREAKING: rename all `http_duration` metrics to `http_request_duration` to match prometheus official naming conventions (See https://prometheus.io/docs/practices/naming/#metric-names).
+
 1.0.1 - 2021-12-22
 
 - FEATURE: add labels to preflight requests

data/README.md

@@ -28,6 +28,7 @@ To learn more see [Instrumenting Rails with Prometheus](https://samsaffron.com/a
   * [Client default labels](#client-default-labels)
   * [Client default host](#client-default-host)
   * [Histogram mode](#histogram-mode)
+  * [Histogram - custom buckets](#histogram-custom-buckets)
 * [Transport concerns](#transport-concerns)
 * [JSON generation and parsing](#json-generation-and-parsing)
 * [Logging](#logging)
@@ -202,13 +203,13 @@ $ bundle exec prometheus_exporter
 
 #### Metrics collected by Rails integration middleware
 
-| Type    | Name                            | Description                                                 |
-| ---     | ---                             | ---                                                         |
-| Counter | `http_requests_total`           | Total HTTP requests from web app                            |
-| Summary | `http_duration_seconds`         | Time spent in HTTP reqs in seconds                          |
-| Summary | `http_redis_duration_seconds`¹  | Time spent in HTTP reqs in Redis, in seconds                |
-| Summary | `http_sql_duration_seconds`²    | Time spent in HTTP reqs in SQL in seconds                   |
-| Summary | `http_queue_duration_seconds`³  | Time spent queueing the request in load balancer in seconds |
+| Type    | Name                                   | Description                                                 |
+| ---     | ---                                    | ---                                                         |
+| Counter | `http_requests_total`                  | Total HTTP requests from web app                            |
+| Summary | `http_request_duration_seconds`        | Time spent in HTTP reqs in seconds                          |
+| Summary | `http_request_redis_duration_seconds`¹ | Time spent in HTTP reqs in Redis, in seconds                |
+| Summary | `http_request_sql_duration_seconds`²   | Time spent in HTTP reqs in SQL in seconds                   |
+| Summary | `http_request_queue_duration_seconds`³ | Time spent queueing the request in load balancer in seconds |
 
 All metrics have a `controller` and an `action` label.
 `http_requests_total` additionally has a (HTTP response) `status` label.
@@ -251,7 +252,7 @@ end
 ```
 That way you won't have all metrics labeled with `controller=other` and `action=other`, but have labels such as
 ```
-ruby_http_duration_seconds{path="/api/v1/teams/:id",method="GET",status="200",quantile="0.99"} 0.009880661998977303
+ruby_http_request_duration_seconds{path="/api/v1/teams/:id",method="GET",status="200",quantile="0.99"} 0.009880661998977303
 ```
 
 ¹) Only available when Redis is used.
@@ -420,6 +421,18 @@ Sometimes the Sidekiq server shuts down before it can send metrics, that were ge
   end
 ```
 
+Custom labels can be added for individual jobs by defining a class method on the job class. These labels will be added to all Sidekiq metrics written by the job:
+
+```ruby
+  class WorkerWithCustomLabels
+    def self.custom_labels
+      { my_label: 'value-here', other_label: 'second-val' }
+    end
+
+    def perform; end
+  end
+```
+
 ##### Metrics collected by Sidekiq Instrumentation
 
 **PrometheusExporter::Instrumentation::Sidekiq**
@@ -891,6 +904,26 @@ In histogram mode, the same metrics will be collected but will be reported as hi
 
 [`histogram_quantile`]: https://prometheus.io/docs/prometheus/latest/querying/functions/#histogram_quantile
 
+### Histogram - custom buckets
+
+By default these buckets will be used:
+```
+[0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5.0, 10.0].freeze
+```
+if this is not enough you can specify `default_buckets` like this:
+```
+Histogram.default_buckets = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 2.5, 3, 4, 5.0, 10.0, 12, 14, 15, 20, 25].freeze
+```
+
+Specfied buckets on the instance  takes precedence over default:
+
+```
+Histogram.default_buckets = [0.005, 0.01, 0,5].freeze
+buckets = [0.1, 0.2, 0.3]
+histogram = Histogram.new('test_bucktets', 'I have specified buckets', buckets: buckets)
+histogram.buckets => [0.1, 0.2, 0.3]
+```
+
 ## Transport concerns
 
 Prometheus Exporter handles transport using a simple HTTP protocol. In multi process mode we avoid needing a large number of HTTP request by using chunked encoding to send metrics. This means that a single HTTP channel can deliver 100s or even 1000s of metrics over a single HTTP session to the `/send-metrics` endpoint. All calls to `send` and `send_json` on the `PrometheusExporter::Client` class are **non-blocking** and batched.

data/lib/prometheus_exporter/instrumentation/sidekiq.rb

@@ -15,16 +15,24 @@ module PrometheusExporter::Instrumentation
       -> (job, ex) do
         job_is_fire_and_forget = job["retry"] == false
 
+        worker_class = Object.const_get(job["class"])
+        worker_custom_labels = self.get_worker_custom_labels(worker_class)
+
         unless job_is_fire_and_forget
           PrometheusExporter::Client.default.send_json(
             type: "sidekiq",
             name: job["class"],
             dead: true,
+            custom_labels: worker_custom_labels
           )
         end
       end
     end
 
+    def self.get_worker_custom_labels(worker_class)
+      worker_class.respond_to?(:custom_labels) ? worker_class.custom_labels : {}
+    end
+
     def initialize(client: nil)
       @client = client || PrometheusExporter::Client.default
     end
@@ -47,7 +55,8 @@ module PrometheusExporter::Instrumentation
         queue: queue,
         success: success,
         shutdown: shutdown,
-        duration: duration
+        duration: duration,
+        custom_labels: self.class.get_worker_custom_labels(worker.class)
       )
     end
 
@@ -69,19 +78,30 @@ module PrometheusExporter::Instrumentation
     end
 
     def get_delayed_name(msg, class_name)
-      # fallback to class_name since we're relying on the internal implementation
-      # of the delayed extensions
-      # https://github.com/mperham/sidekiq/blob/master/lib/sidekiq/extensions/class_methods.rb
       begin
+        # fallback to class_name since we're relying on the internal implementation
+        # of the delayed extensions
+        # https://github.com/mperham/sidekiq/blob/master/lib/sidekiq/extensions/class_methods.rb
         (target, method_name, _args) = YAML.load(msg['args'].first) # rubocop:disable Security/YAMLLoad
         if target.class == Class
           "#{target.name}##{method_name}"
         else
           "#{target.class.name}##{method_name}"
         end
-      rescue
-        class_name
+      rescue Psych::DisallowedClass, ArgumentError
+        parsed = Psych.parse(msg['args'].first)
+        children = parsed.root.children
+        target = (children[0].value || children[0].tag).sub('!', '')
+        method_name = (children[1].value || children[1].tag).sub(':', '')
+
+        if target && method_name
+          "#{target}##{method_name}"
+        else
+          class_name
+        end
       end
+    rescue
+      class_name
     end
   end
 end

data/lib/prometheus_exporter/metric/base.rb

@@ -106,15 +106,8 @@ module PrometheusExporter::Metric
       end
     end
 
-    # when we drop Ruby 2.3 we can drop this
-    if "".respond_to? :match?
-      def needs_escape?(str)
-        str.match?(/[\n"\\]/m)
-      end
-    else
-      def needs_escape?(str)
-        !!str.match(/[\n"\\]/m)
-      end
+    def needs_escape?(str)
+      str.match?(/[\n"\\]/m)
     end
 
   end

data/lib/prometheus_exporter/metric/histogram.rb

@@ -5,9 +5,21 @@ module PrometheusExporter::Metric
 
     DEFAULT_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5.0, 10.0].freeze
 
+    @default_buckets = nil if !defined?(@default_buckets)
+
+    def self.default_buckets
+      @default_buckets || DEFAULT_BUCKETS
+    end
+
+    def self.default_buckets=(buckets)
+      @default_buckets = buckets
+    end
+
+    attr_reader :buckets
+
     def initialize(name, help, opts = {})
       super(name, help)
-      @buckets = (opts[:buckets] || DEFAULT_BUCKETS).sort.reverse
+      @buckets = (opts[:buckets] || self.class.default_buckets).sort.reverse
       reset!
     end
 

data/lib/prometheus_exporter/middleware.rb

@@ -36,11 +36,12 @@ class PrometheusExporter::Middleware
 
     result
   ensure
-
+    status = (result && result[0]) || -1
     obj = {
       type: "web",
       timings: info,
       queue_time: queue_time,
+      status: status,
       default_labels: default_labels(env, result)
     }
     labels = custom_labels(env)
@@ -52,7 +53,6 @@ class PrometheusExporter::Middleware
   end
 
   def default_labels(env, result)
-    status = (result && result[0]) || -1
     params = env["action_dispatch.request.parameters"]
     action = controller = nil
     if params
@@ -67,8 +67,7 @@ class PrometheusExporter::Middleware
 
     {
       action: action || "other",
-      controller: controller || "other",
-      status: status
+      controller: controller || "other"
     }
   end
 

data/lib/prometheus_exporter/server/web_collector.rb

@@ -5,10 +5,10 @@ module PrometheusExporter::Server
     def initialize
       @metrics = {}
       @http_requests_total = nil
-      @http_duration_seconds = nil
-      @http_redis_duration_seconds = nil
-      @http_sql_duration_seconds = nil
-      @http_queue_duration_seconds = nil
+      @http_request_duration_seconds = nil
+      @http_request_redis_duration_seconds = nil
+      @http_request_sql_duration_seconds = nil
+      @http_request_queue_duration_seconds = nil
     end
 
     def type
@@ -33,23 +33,23 @@ module PrometheusExporter::Server
           "Total HTTP requests from web app."
         )
 
-        @metrics["http_duration_seconds"] = @http_duration_seconds = PrometheusExporter::Metric::Base.default_aggregation.new(
-          "http_duration_seconds",
+        @metrics["http_request_duration_seconds"] = @http_request_duration_seconds = PrometheusExporter::Metric::Base.default_aggregation.new(
+          "http_request_duration_seconds",
           "Time spent in HTTP reqs in seconds."
         )
 
-        @metrics["http_redis_duration_seconds"] = @http_redis_duration_seconds = PrometheusExporter::Metric::Base.default_aggregation.new(
-          "http_redis_duration_seconds",
+        @metrics["http_request_redis_duration_seconds"] = @http_request_redis_duration_seconds = PrometheusExporter::Metric::Base.default_aggregation.new(
+          "http_request_redis_duration_seconds",
           "Time spent in HTTP reqs in Redis, in seconds."
         )
 
-        @metrics["http_sql_duration_seconds"] = @http_sql_duration_seconds = PrometheusExporter::Metric::Base.default_aggregation.new(
-          "http_sql_duration_seconds",
+        @metrics["http_request_sql_duration_seconds"] = @http_request_sql_duration_seconds = PrometheusExporter::Metric::Base.default_aggregation.new(
+          "http_request_sql_duration_seconds",
           "Time spent in HTTP reqs in SQL in seconds."
         )
 
-        @metrics["http_queue_duration_seconds"] = @http_queue_duration_seconds = PrometheusExporter::Metric::Base.default_aggregation.new(
-          "http_queue_duration_seconds",
+        @metrics["http_request_queue_duration_seconds"] = @http_request_queue_duration_seconds = PrometheusExporter::Metric::Base.default_aggregation.new(
+          "http_request_queue_duration_seconds",
           "Time spent queueing the request in load balancer in seconds."
         )
       end
@@ -60,19 +60,19 @@ module PrometheusExporter::Server
       custom_labels = obj['custom_labels']
       labels = custom_labels.nil? ? default_labels : default_labels.merge(custom_labels)
 
-      @http_requests_total.observe(1, labels)
+      @http_requests_total.observe(1, labels.merge(status: obj["status"]))
 
       if timings = obj["timings"]
-        @http_duration_seconds.observe(timings["total_duration"], labels)
+        @http_request_duration_seconds.observe(timings["total_duration"], labels)
         if redis = timings["redis"]
-          @http_redis_duration_seconds.observe(redis["duration"], labels)
+          @http_request_redis_duration_seconds.observe(redis["duration"], labels)
         end
         if sql = timings["sql"]
-          @http_sql_duration_seconds.observe(sql["duration"], labels)
+          @http_request_sql_duration_seconds.observe(sql["duration"], labels)
         end
       end
       if queue_time = obj["queue_time"]
-        @http_queue_duration_seconds.observe(queue_time, labels)
+        @http_request_queue_duration_seconds.observe(queue_time, labels)
       end
     end
   end

data/lib/prometheus_exporter/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PrometheusExporter
-  VERSION = '1.0.1'
+  VERSION = '2.0.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: prometheus_exporter
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 2.0.0
 platform: ruby
 authors:
 - Sam Saffron
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-12-22 00:00:00.000000000 Z
+date: 2022-02-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: webrick