prometheus_exporter 0.7.0 → 2.3.0

data/lib/prometheus_exporter/metric/histogram.rb

@@ -2,12 +2,23 @@
 
 module PrometheusExporter::Metric
   class Histogram < Base
-
     DEFAULT_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5.0, 10.0].freeze
 
+    @default_buckets = nil if !defined?(@default_buckets)
+
+    def self.default_buckets
+      @default_buckets || DEFAULT_BUCKETS
+    end
+
+    def self.default_buckets=(buckets)
+      @default_buckets = buckets
+    end
+
+    attr_reader :buckets
+
     def initialize(name, help, opts = {})
       super(name, help)
-      @buckets = (opts[:buckets] || DEFAULT_BUCKETS).sort.reverse
+      @buckets = (opts[:buckets] || self.class.default_buckets).sort
       reset!
     end
 
@@ -45,11 +56,11 @@ module PrometheusExporter::Metric
         first = false
         count = @counts[labels]
         sum = @sums[labels]
-        text << "#{prefix(@name)}_bucket#{labels_text(with_bucket(labels, "+Inf"))} #{count}\n"
         @buckets.each do |bucket|
           value = @observations[labels][bucket]
           text << "#{prefix(@name)}_bucket#{labels_text(with_bucket(labels, bucket.to_s))} #{value}\n"
         end
+        text << "#{prefix(@name)}_bucket#{labels_text(with_bucket(labels, "+Inf"))} #{count}\n"
         text << "#{prefix(@name)}_count#{labels_text(labels)} #{count}\n"
         text << "#{prefix(@name)}_sum#{labels_text(labels)} #{sum}"
       end
@@ -79,7 +90,7 @@ module PrometheusExporter::Metric
     end
 
     def fill_buckets(value, buckets)
-      @buckets.each do |b|
+      @buckets.reverse_each do |b|
         break if value > b
         buckets[b] += 1
       end
@@ -88,6 +99,5 @@ module PrometheusExporter::Metric
     def with_bucket(labels, bucket)
       labels.merge("le" => bucket)
     end
-
   end
 end

data/lib/prometheus_exporter/metric/summary.rb

@@ -2,7 +2,6 @@
 
 module PrometheusExporter::Metric
   class Summary < Base
-
     DEFAULT_QUANTILES = [0.99, 0.9, 0.5, 0.1, 0.01]
     ROTATE_AGE = 120
 
@@ -49,9 +48,7 @@ module PrometheusExporter::Metric
       result = {}
 
       if length > 0
-        @quantiles.each do |quantile|
-          result[quantile] = sorted[(length * quantile).ceil - 1]
-        end
+        @quantiles.each { |quantile| result[quantile] = sorted[(length * quantile).ceil - 1] }
       end
 
       result
@@ -61,12 +58,9 @@ module PrometheusExporter::Metric
       buffer = @buffers[@current_buffer]
 
       result = {}
-      buffer.each do |labels, raw_data|
-        result[labels] = calculate_quantiles(raw_data)
-      end
+      buffer.each { |labels, raw_data| result[labels] = calculate_quantiles(raw_data) }
 
       result
-
     end
 
     def metric_text
@@ -87,8 +81,8 @@ module PrometheusExporter::Metric
 
     # makes sure we have storage
     def ensure_summary(labels)
-      @buffers[0][labels] ||=  []
-      @buffers[1][labels] ||=  []
+      @buffers[0][labels] ||= []
+      @buffers[1][labels] ||= []
       @sums[labels] ||= 0.0
       @counts[labels] ||= 0
       nil
@@ -97,9 +91,7 @@ module PrometheusExporter::Metric
     def rotate_if_needed
       if (now = Process.clock_gettime(Process::CLOCK_MONOTONIC)) > (@last_rotated + ROTATE_AGE)
         @last_rotated = now
-        @buffers[@current_buffer].each do |labels, raw|
-          raw.clear
-        end
+        @buffers[@current_buffer].each { |labels, raw| raw.clear }
         @current_buffer = @current_buffer == 0 ? 1 : 0
       end
       nil
@@ -116,6 +108,5 @@ module PrometheusExporter::Metric
       @sums[labels] += value
       @counts[labels] += 1
     end
-
   end
 end

data/lib/prometheus_exporter/middleware.rb

@@ -1,28 +1,51 @@
 # frozen_string_literal: true
 
-require 'prometheus_exporter/instrumentation/method_profiler'
-require 'prometheus_exporter/client'
+require "prometheus_exporter/instrumentation/method_profiler"
+require "prometheus_exporter/client"
 
 class PrometheusExporter::Middleware
   MethodProfiler = PrometheusExporter::Instrumentation::MethodProfiler
 
-  def initialize(app, config = { instrument: true, client: nil })
+  def initialize(app, config = { instrument: :alias_method, client: nil })
     @app = app
     @client = config[:client] || PrometheusExporter::Client.default
 
     if config[:instrument]
-      if defined? Redis::Client
-        MethodProfiler.patch(Redis::Client, [:call, :call_pipeline], :redis)
+      apply_redis_client_middleware! if defined?(RedisClient)
+
+      if defined?(Redis::VERSION) && (Gem::Version.new(Redis::VERSION) >= Gem::Version.new("5.0.0"))
+        # redis 5 support handled via RedisClient
+      elsif defined?(Redis::Client)
+        MethodProfiler.patch(
+          Redis::Client,
+          %i[call call_pipeline],
+          :redis,
+          instrument: config[:instrument],
+        )
       end
-      if defined? PG::Connection
-        MethodProfiler.patch(PG::Connection, [
-          :exec, :async_exec, :exec_prepared, :send_query_prepared, :query
-        ], :sql)
+
+      if defined?(PG::Connection)
+        MethodProfiler.patch(
+          PG::Connection,
+          %i[exec async_exec exec_prepared exec_params send_query_prepared query],
+          :sql,
+          instrument: config[:instrument],
+        )
+      end
+
+      if defined?(Mysql2::Client)
+        MethodProfiler.patch(Mysql2::Client, [:query], :sql, instrument: config[:instrument])
+        MethodProfiler.patch(Mysql2::Statement, [:execute], :sql, instrument: config[:instrument])
+        MethodProfiler.patch(Mysql2::Result, [:each], :sql, instrument: config[:instrument])
       end
-      if defined? Mysql2::Client
-        MethodProfiler.patch(Mysql2::Client, [:query], :sql)
-        MethodProfiler.patch(Mysql2::Statement, [:execute], :sql)
-        MethodProfiler.patch(Mysql2::Result, [:each], :sql)
+
+      if defined?(Dalli::Client)
+        MethodProfiler.patch(
+          Dalli::Client,
+          %i[delete fetch get add set],
+          :memcache,
+          instrument: config[:instrument],
+        )
       end
     end
   end
@@ -36,35 +59,34 @@ class PrometheusExporter::Middleware
 
     result
   ensure
-
+    status = (result && result[0]) || -1
     obj = {
       type: "web",
       timings: info,
       queue_time: queue_time,
-      default_labels: default_labels(env, result)
+      status: status,
+      default_labels: default_labels(env, result),
     }
     labels = custom_labels(env)
-    if labels
-      obj = obj.merge(custom_labels: labels)
-    end
+    obj = obj.merge(custom_labels: labels) if labels
 
     @client.send_json(obj)
   end
 
   def default_labels(env, result)
-    status = (result && result[0]) || -1
-    params = env["action_dispatch.request.parameters"]
+    controller_instance = env["action_controller.instance"]
     action = controller = nil
-    if params
-      action = params["action"]
-      controller = params["controller"]
+    if controller_instance
+      action = controller_instance.action_name
+      controller = controller_instance.controller_name
+    elsif (cors = env["rack.cors"]) && cors.respond_to?(:preflight?) && cors.preflight?
+      # if the Rack CORS Middleware identifies the request as a preflight request,
+      # the stack doesn't get to the point where controllers/actions are defined
+      action = "preflight"
+      controller = "preflight"
     end
 
-    {
-      action: action || "other",
-      controller: controller || "other",
-      status: status
-    }
+    { action: action || "other", controller: controller || "other" }
   end
 
   # allows subclasses to add custom labels based on env
@@ -90,19 +112,31 @@ class PrometheusExporter::Middleware
     Process.clock_gettime(Process::CLOCK_REALTIME)
   end
 
-  # get the content of the x-queue-start or x-request-start header
+  # determine queue start from well-known trace headers
   def queue_start(env)
-    value = env['HTTP_X_REQUEST_START'] || env['HTTP_X_QUEUE_START']
-    unless value.nil? || value == ''
-      convert_header_to_ms(value.to_s)
+    # get the content of the x-queue-start or x-request-start header
+    value = env["HTTP_X_REQUEST_START"] || env["HTTP_X_QUEUE_START"]
+    unless value.nil? || value == ""
+      # nginx returns time as milliseconds with 3 decimal places
+      # apache returns time as microseconds without decimal places
+      # this method takes care to convert both into a proper second + fractions timestamp
+      value = value.to_s.gsub(/t=|\./, "")
+      return "#{value[0, 10]}.#{value[10, 13]}".to_f
     end
+
+    # get the content of the x-amzn-trace-id header
+    # see also: https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-request-tracing.html
+    value = env["HTTP_X_AMZN_TRACE_ID"]
+    value&.split("Root=")&.last&.split("-")&.fetch(1)&.to_i(16)
+  end
+
+  private
+
+  module RedisInstrumenter
+    MethodProfiler.define_methods_on_module(self, %w[call call_pipelined], "redis")
   end
 
-  # nginx returns time as milliseconds with 3 decimal places
-  # apache returns time as microseconds without decimal places
-  # this method takes care to convert both into a proper second + fractions timestamp
-  def convert_header_to_ms(str)
-    str = str.gsub(/t=|\./, '')
-    "#{str[0, 10]}.#{str[10, 13]}".to_f
+  def apply_redis_client_middleware!
+    RedisClient.register(RedisInstrumenter)
   end
 end

data/lib/prometheus_exporter/server/active_record_collector.rb

@@ -2,18 +2,24 @@
 
 module PrometheusExporter::Server
   class ActiveRecordCollector < TypeCollector
-    MAX_ACTIVERECORD_METRIC_AGE = 60
+    MAX_METRIC_AGE = 60
+
     ACTIVE_RECORD_GAUGES = {
       connections: "Total connections in pool",
       busy: "Connections in use in pool",
       dead: "Dead connections in pool",
       idle: "Idle connections in pool",
       waiting: "Connection requests waiting",
-      size: "Maximum allowed connection pool size"
+      size: "Maximum allowed connection pool size",
     }
 
     def initialize
-      @active_record_metrics = []
+      @active_record_metrics = MetricsContainer.new(ttl: MAX_METRIC_AGE)
+      @active_record_metrics.filter = ->(new_metric, old_metric) do
+        new_metric["pid"] == old_metric["pid"] &&
+          new_metric["hostname"] == old_metric["hostname"] &&
+          new_metric["metric_labels"]["pool_name"] == old_metric["metric_labels"]["pool_name"]
+      end
     end
 
     def type
@@ -26,13 +32,18 @@ module PrometheusExporter::Server
       metrics = {}
 
       @active_record_metrics.map do |m|
-        metric_key = (m["metric_labels"] || {}).merge("pid" => m["pid"])
+        metric_key =
+          (m["metric_labels"] || {}).merge("pid" => m["pid"], "hostname" => m["hostname"])
         metric_key.merge!(m["custom_labels"]) if m["custom_labels"]
 
         ACTIVE_RECORD_GAUGES.map do |k, help|
           k = k.to_s
           if v = m[k]
-            g = metrics[k] ||= PrometheusExporter::Metric::Gauge.new("active_record_connection_pool_#{k}", help)
+            g =
+              metrics[k] ||= PrometheusExporter::Metric::Gauge.new(
+                "active_record_connection_pool_#{k}",
+                help,
+              )
             g.observe(v, metric_key)
           end
         end
@@ -42,15 +53,6 @@ module PrometheusExporter::Server
     end
 
     def collect(obj)
-      now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
-
-      obj["created_at"] = now
-
-      @active_record_metrics.delete_if do |current|
-        (obj["pid"] == current["pid"] && obj["hostname"] == current["hostname"]) ||
-          (current["created_at"] + MAX_ACTIVERECORD_METRIC_AGE < now)
-      end
-
       @active_record_metrics << obj
     end
   end

data/lib/prometheus_exporter/server/collector.rb

@@ -1,10 +1,13 @@
 # frozen_string_literal: true
 
-module PrometheusExporter::Server
+require "logger"
 
+module PrometheusExporter::Server
   class Collector < CollectorBase
+    attr_reader :logger
 
-    def initialize(json_serializer: nil)
+    def initialize(json_serializer: nil, logger: Logger.new(STDERR))
+      @logger = logger
       @process_metrics = []
       @metrics = {}
       @mutex = Mutex.new
@@ -14,12 +17,16 @@ module PrometheusExporter::Server
       register_collector(ProcessCollector.new)
       register_collector(SidekiqCollector.new)
       register_collector(SidekiqQueueCollector.new)
+      register_collector(SidekiqProcessCollector.new)
+      register_collector(SidekiqStatsCollector.new)
       register_collector(DelayedJobCollector.new)
       register_collector(PumaCollector.new)
       register_collector(HutchCollector.new)
       register_collector(UnicornCollector.new)
       register_collector(ActiveRecordCollector.new)
       register_collector(ShoryukenCollector.new)
+      register_collector(ResqueCollector.new)
+      register_collector(GoodJobCollector.new)
     end
 
     def register_collector(collector)
@@ -36,19 +43,17 @@ module PrometheusExporter::Server
           collector.collect(obj)
         else
           metric = @metrics[obj["name"]]
-          if !metric
-            metric = register_metric_unsafe(obj)
-          end
+          metric = register_metric_unsafe(obj) if !metric
+
+          next unless metric
 
           keys = obj["keys"] || {}
-          if obj["custom_labels"]
-            keys = obj["custom_labels"].merge(keys)
-          end
+          keys = obj["custom_labels"].merge(keys) if obj["custom_labels"]
 
           case obj["prometheus_exporter_action"]
-          when 'increment'
+          when "increment"
             metric.increment(keys, obj["value"])
-          when 'decrement'
+          when "decrement"
             metric.decrement(keys, obj["value"])
           else
             metric.observe(obj["value"], keys)
@@ -59,15 +64,14 @@ module PrometheusExporter::Server
 
     def prometheus_metrics_text
       @mutex.synchronize do
-        (@metrics.values + @collectors.values.map(&:metrics).flatten)
-          .map(&:to_prometheus_text).join("\n")
+        (@metrics.values + @collectors.values.map(&:metrics).flatten).map(
+          &:to_prometheus_text
+        ).join("\n")
       end
     end
 
     def register_metric(metric)
-      @mutex.synchronize do
-        @metrics[metric.name] = metric
-      end
+      @mutex.synchronize { @metrics[metric.name] = metric }
     end
 
     protected
@@ -77,6 +81,11 @@ module PrometheusExporter::Server
       help = obj["help"]
       opts = symbolize_keys(obj["opts"] || {})
 
+      if !name
+        logger.warn "failed to register metric due to empty name #{obj}"
+        return
+      end
+
       metric =
         case obj["type"]
         when "gauge"
@@ -92,12 +101,15 @@ module PrometheusExporter::Server
       if metric
         @metrics[name] = metric
       else
-        STDERR.puts "failed to register metric #{obj}"
+        logger.warn "failed to register metric #{obj}"
       end
     end
 
     def symbolize_keys(hash)
-      hash.inject({}) { |memo, k| memo[k.first.to_sym] = k.last; memo }
+      hash.inject({}) do |memo, k|
+        memo[k.first.to_sym] = k.last
+        memo
+      end
     end
   end
 end

data/lib/prometheus_exporter/server/collector_base.rb

@@ -1,10 +1,8 @@
 # frozen_string_literal: true
 
 module PrometheusExporter::Server
-
   # minimal interface to implement a customer collector
   class CollectorBase
-
     # called each time a string is delivered from the web
     def process(str)
     end

data/lib/prometheus_exporter/server/delayed_job_collector.rb

@@ -5,6 +5,7 @@ module PrometheusExporter::Server
     def initialize
       @delayed_jobs_total = nil
       @delayed_job_duration_seconds = nil
+      @delayed_job_latency_seconds_total = nil
       @delayed_jobs_total = nil
       @delayed_failed_jobs_total = nil
       @delayed_jobs_max_attempts_reached_total = nil
@@ -19,28 +20,49 @@ module PrometheusExporter::Server
     end
 
     def collect(obj)
-      default_labels = { job_name: obj['name'] }
-      custom_labels = obj['custom_labels']
-      labels = custom_labels.nil? ? default_labels : default_labels.merge(custom_labels)
+      custom_labels = obj["custom_labels"] || {}
+      gauge_labels = { queue_name: obj["queue_name"] }.merge(custom_labels)
+      counter_labels = gauge_labels.merge(job_name: obj["name"])
 
       ensure_delayed_job_metrics
-      @delayed_job_duration_seconds.observe(obj["duration"], labels)
-      @delayed_jobs_total.observe(1, labels)
-      @delayed_failed_jobs_total.observe(1, labels) if !obj["success"]
-      @delayed_jobs_max_attempts_reached_total.observe(1) if obj["attempts"] >= obj["max_attempts"]
-      @delayed_job_duration_seconds_summary.observe(obj["duration"])
-      @delayed_job_duration_seconds_summary.observe(obj["duration"], status: "success") if obj["success"]
-      @delayed_job_duration_seconds_summary.observe(obj["duration"], status: "failed")  if !obj["success"]
-      @delayed_job_attempts_summary.observe(obj["attempts"]) if obj["success"]
-      @delayed_jobs_enqueued.observe(obj["enqueued"])
-      @delayed_jobs_pending.observe(obj["pending"])
+      @delayed_job_duration_seconds.observe(obj["duration"], counter_labels)
+      @delayed_job_latency_seconds_total.observe(obj["latency"], counter_labels)
+      @delayed_jobs_total.observe(1, counter_labels)
+      @delayed_failed_jobs_total.observe(1, counter_labels) if !obj["success"]
+      if obj["attempts"] >= obj["max_attempts"]
+        @delayed_jobs_max_attempts_reached_total.observe(1, counter_labels)
+      end
+      @delayed_job_duration_seconds_summary.observe(obj["duration"], counter_labels)
+      if obj["success"]
+        @delayed_job_duration_seconds_summary.observe(
+          obj["duration"],
+          counter_labels.merge(status: "success"),
+        )
+      end
+      if !obj["success"]
+        @delayed_job_duration_seconds_summary.observe(
+          obj["duration"],
+          counter_labels.merge(status: "failed"),
+        )
+      end
+      @delayed_job_attempts_summary.observe(obj["attempts"], counter_labels) if obj["success"]
+      @delayed_jobs_enqueued.observe(obj["enqueued"], gauge_labels)
+      @delayed_jobs_pending.observe(obj["pending"], gauge_labels)
     end
 
     def metrics
       if @delayed_jobs_total
-        [@delayed_job_duration_seconds, @delayed_jobs_total, @delayed_failed_jobs_total,
-         @delayed_jobs_max_attempts_reached_total, @delayed_job_duration_seconds_summary, @delayed_job_attempts_summary,
-         @delayed_jobs_enqueued, @delayed_jobs_pending]
+        [
+          @delayed_job_duration_seconds,
+          @delayed_job_latency_seconds_total,
+          @delayed_jobs_total,
+          @delayed_failed_jobs_total,
+          @delayed_jobs_max_attempts_reached_total,
+          @delayed_job_duration_seconds_summary,
+          @delayed_job_attempts_summary,
+          @delayed_jobs_enqueued,
+          @delayed_jobs_pending,
+        ]
       else
         []
       end
@@ -50,38 +72,59 @@ module PrometheusExporter::Server
 
     def ensure_delayed_job_metrics
       if !@delayed_jobs_total
-
         @delayed_job_duration_seconds =
-        PrometheusExporter::Metric::Counter.new(
-          "delayed_job_duration_seconds", "Total time spent in delayed jobs.")
+          PrometheusExporter::Metric::Counter.new(
+            "delayed_job_duration_seconds",
+            "Total time spent in delayed jobs.",
+          )
+
+        @delayed_job_latency_seconds_total =
+          PrometheusExporter::Metric::Counter.new(
+            "delayed_job_latency_seconds_total",
+            "Total delayed jobs latency.",
+          )
 
         @delayed_jobs_total =
-        PrometheusExporter::Metric::Counter.new(
-          "delayed_jobs_total", "Total number of delayed jobs executed.")
+          PrometheusExporter::Metric::Counter.new(
+            "delayed_jobs_total",
+            "Total number of delayed jobs executed.",
+          )
 
         @delayed_jobs_enqueued =
-        PrometheusExporter::Metric::Gauge.new(
-          "delayed_jobs_enqueued", "Number of enqueued delayed jobs.")
+          PrometheusExporter::Metric::Gauge.new(
+            "delayed_jobs_enqueued",
+            "Number of enqueued delayed jobs.",
+          )
 
         @delayed_jobs_pending =
-        PrometheusExporter::Metric::Gauge.new(
-          "delayed_jobs_pending", "Number of pending delayed jobs.")
+          PrometheusExporter::Metric::Gauge.new(
+            "delayed_jobs_pending",
+            "Number of pending delayed jobs.",
+          )
 
         @delayed_failed_jobs_total =
-        PrometheusExporter::Metric::Counter.new(
-          "delayed_failed_jobs_total", "Total number failed delayed jobs executed.")
+          PrometheusExporter::Metric::Counter.new(
+            "delayed_failed_jobs_total",
+            "Total number failed delayed jobs executed.",
+          )
 
         @delayed_jobs_max_attempts_reached_total =
-            PrometheusExporter::Metric::Counter.new(
-                "delayed_jobs_max_attempts_reached_total", "Total number of delayed jobs that reached max attempts.")
+          PrometheusExporter::Metric::Counter.new(
+            "delayed_jobs_max_attempts_reached_total",
+            "Total number of delayed jobs that reached max attempts.",
+          )
 
         @delayed_job_duration_seconds_summary =
-            PrometheusExporter::Metric::Summary.new("delayed_job_duration_seconds_summary",
-                                                    "Summary of the time it takes jobs to execute.")
+          PrometheusExporter::Metric::Base.default_aggregation.new(
+            "delayed_job_duration_seconds_summary",
+            "Summary of the time it takes jobs to execute.",
+          )
 
         @delayed_job_attempts_summary =
-            PrometheusExporter::Metric::Summary.new("delayed_job_attempts_summary",
-                                                    "Summary of the amount of attempts it takes delayed jobs to succeed.")
+          PrometheusExporter::Metric::Base.default_aggregation.new(
+            "delayed_job_attempts_summary",
+            "Summary of the amount of attempts it takes delayed jobs to succeed.",
+          )
       end
     end
   end

data/lib/prometheus_exporter/server/good_job_collector.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module PrometheusExporter::Server
+  class GoodJobCollector < TypeCollector
+    MAX_METRIC_AGE = 30
+    GOOD_JOB_GAUGES = {
+      scheduled: "Total number of scheduled GoodJob jobs.",
+      retried: "Total number of retried GoodJob jobs.",
+      queued: "Total number of queued GoodJob jobs.",
+      running: "Total number of running GoodJob jobs.",
+      finished: "Total number of finished GoodJob jobs.",
+      succeeded: "Total number of succeeded GoodJob jobs.",
+      discarded: "Total number of discarded GoodJob jobs.",
+    }
+
+    def initialize
+      @good_job_metrics = MetricsContainer.new(ttl: MAX_METRIC_AGE)
+      @gauges = {}
+    end
+
+    def type
+      "good_job"
+    end
+
+    def metrics
+      return [] if good_job_metrics.length == 0
+
+      good_job_metrics.map do |metric|
+        labels = metric.fetch("custom_labels", {})
+
+        GOOD_JOB_GAUGES.map do |name, help|
+          value = metric[name.to_s]
+
+          if value
+            gauge = gauges[name] ||= PrometheusExporter::Metric::Gauge.new("good_job_#{name}", help)
+            gauge.observe(value, labels)
+          end
+        end
+      end
+
+      gauges.values
+    end
+
+    def collect(object)
+      @good_job_metrics << object
+    end
+
+    private
+
+    attr_reader :good_job_metrics, :gauges
+  end
+end