prometheus_exporter 0.7.0 → 2.1.0

data/lib/prometheus_exporter/server/delayed_job_collector.rb

@@ -5,6 +5,7 @@ module PrometheusExporter::Server
     def initialize
       @delayed_jobs_total = nil
       @delayed_job_duration_seconds = nil
+      @delayed_job_latency_seconds_total = nil
       @delayed_jobs_total = nil
       @delayed_failed_jobs_total = nil
       @delayed_jobs_max_attempts_reached_total = nil
@@ -19,26 +20,27 @@ module PrometheusExporter::Server
     end
 
     def collect(obj)
-      default_labels = { job_name: obj['name'] }
-      custom_labels = obj['custom_labels']
-      labels = custom_labels.nil? ? default_labels : default_labels.merge(custom_labels)
+      custom_labels = obj['custom_labels'] || {}
+      gauge_labels = { queue_name: obj['queue_name'] }.merge(custom_labels)
+      counter_labels = gauge_labels.merge(job_name: obj['name'])
 
       ensure_delayed_job_metrics
-      @delayed_job_duration_seconds.observe(obj["duration"], labels)
-      @delayed_jobs_total.observe(1, labels)
-      @delayed_failed_jobs_total.observe(1, labels) if !obj["success"]
-      @delayed_jobs_max_attempts_reached_total.observe(1) if obj["attempts"] >= obj["max_attempts"]
-      @delayed_job_duration_seconds_summary.observe(obj["duration"])
-      @delayed_job_duration_seconds_summary.observe(obj["duration"], status: "success") if obj["success"]
-      @delayed_job_duration_seconds_summary.observe(obj["duration"], status: "failed")  if !obj["success"]
-      @delayed_job_attempts_summary.observe(obj["attempts"]) if obj["success"]
-      @delayed_jobs_enqueued.observe(obj["enqueued"])
-      @delayed_jobs_pending.observe(obj["pending"])
+      @delayed_job_duration_seconds.observe(obj["duration"], counter_labels)
+      @delayed_job_latency_seconds_total.observe(obj["latency"], counter_labels)
+      @delayed_jobs_total.observe(1, counter_labels)
+      @delayed_failed_jobs_total.observe(1, counter_labels) if !obj["success"]
+      @delayed_jobs_max_attempts_reached_total.observe(1, counter_labels) if obj["attempts"] >= obj["max_attempts"]
+      @delayed_job_duration_seconds_summary.observe(obj["duration"], counter_labels)
+      @delayed_job_duration_seconds_summary.observe(obj["duration"], counter_labels.merge(status: "success")) if obj["success"]
+      @delayed_job_duration_seconds_summary.observe(obj["duration"], counter_labels.merge(status: "failed"))  if !obj["success"]
+      @delayed_job_attempts_summary.observe(obj["attempts"], counter_labels) if obj["success"]
+      @delayed_jobs_enqueued.observe(obj["enqueued"], gauge_labels)
+      @delayed_jobs_pending.observe(obj["pending"], gauge_labels)
     end
 
     def metrics
       if @delayed_jobs_total
-        [@delayed_job_duration_seconds, @delayed_jobs_total, @delayed_failed_jobs_total,
+        [@delayed_job_duration_seconds, @delayed_job_latency_seconds_total, @delayed_jobs_total, @delayed_failed_jobs_total,
          @delayed_jobs_max_attempts_reached_total, @delayed_job_duration_seconds_summary, @delayed_job_attempts_summary,
          @delayed_jobs_enqueued, @delayed_jobs_pending]
       else
@@ -55,6 +57,10 @@ module PrometheusExporter::Server
         PrometheusExporter::Metric::Counter.new(
           "delayed_job_duration_seconds", "Total time spent in delayed jobs.")
 
+        @delayed_job_latency_seconds_total =
+        PrometheusExporter::Metric::Counter.new(
+          "delayed_job_latency_seconds_total", "Total delayed jobs latency.")
+
         @delayed_jobs_total =
         PrometheusExporter::Metric::Counter.new(
           "delayed_jobs_total", "Total number of delayed jobs executed.")
@@ -76,12 +82,12 @@ module PrometheusExporter::Server
                 "delayed_jobs_max_attempts_reached_total", "Total number of delayed jobs that reached max attempts.")
 
         @delayed_job_duration_seconds_summary =
-            PrometheusExporter::Metric::Summary.new("delayed_job_duration_seconds_summary",
-                                                    "Summary of the time it takes jobs to execute.")
+            PrometheusExporter::Metric::Base.default_aggregation.new("delayed_job_duration_seconds_summary",
+                                                                     "Summary of the time it takes jobs to execute.")
 
         @delayed_job_attempts_summary =
-            PrometheusExporter::Metric::Summary.new("delayed_job_attempts_summary",
-                                                    "Summary of the amount of attempts it takes delayed jobs to succeed.")
+            PrometheusExporter::Metric::Base.default_aggregation.new("delayed_job_attempts_summary",
+                                                                     "Summary of the amount of attempts it takes delayed jobs to succeed.")
       end
     end
   end

data/lib/prometheus_exporter/server/good_job_collector.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module PrometheusExporter::Server
+  class GoodJobCollector < TypeCollector
+    MAX_METRIC_AGE = 30
+    GOOD_JOB_GAUGES = {
+      scheduled: "Total number of scheduled GoodJob jobs.",
+      retried: "Total number of retried GoodJob jobs.",
+      queued: "Total number of queued GoodJob jobs.",
+      running: "Total number of running GoodJob jobs.",
+      finished: "Total number of finished GoodJob jobs.",
+      succeeded: "Total number of succeeded GoodJob jobs.",
+      discarded: "Total number of discarded GoodJob jobs."
+    }
+
+    def initialize
+      @good_job_metrics = MetricsContainer.new(ttl: MAX_METRIC_AGE)
+      @gauges = {}
+    end
+
+    def type
+      "good_job"
+    end
+
+    def metrics
+      return [] if good_job_metrics.length == 0
+
+      good_job_metrics.map do |metric|
+        labels = metric.fetch("custom_labels", {})
+
+        GOOD_JOB_GAUGES.map do |name, help|
+          value = metric[name.to_s]
+
+          if value
+            gauge = gauges[name] ||= PrometheusExporter::Metric::Gauge.new("good_job_#{name}", help)
+            gauge.observe(value, labels)
+          end
+        end
+      end
+
+      gauges.values
+    end
+
+    def collect(object)
+      @good_job_metrics << object
+    end
+
+    private
+
+    attr_reader :good_job_metrics, :gauges
+  end
+end

data/lib/prometheus_exporter/server/metrics_container.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module PrometheusExporter::Server
+  class MetricsContainer
+    METRIC_MAX_AGE = 60
+    METRIC_EXPIRE_ATTR = "_expire_at"
+
+    attr_reader :data, :ttl
+    attr_accessor :filter
+
+    def initialize(ttl: METRIC_MAX_AGE, expire_attr: METRIC_EXPIRE_ATTR, filter: nil)
+      @data          = []
+      @ttl           = ttl
+      @expire_attr   = expire_attr
+      @filter        = filter
+    end
+
+    def <<(obj)
+      now = get_time
+      obj[@expire_attr] = now + @ttl
+
+      expire(time: now, new_metric: obj)
+
+      @data << obj
+      @data
+    end
+
+    def [](key)
+      @data.tap { expire }[key]
+    end
+
+    def size(&blk)
+      wrap_expire(:size, &blk)
+    end
+    alias_method :length, :size
+
+    def map(&blk)
+      wrap_expire(:map, &blk)
+    end
+
+    def each(&blk)
+      wrap_expire(:each, &blk)
+    end
+
+    def expire(time: nil, new_metric: nil)
+      time ||= get_time
+
+      @data.delete_if do |metric|
+        expired = metric[@expire_attr] < time
+        expired ||= filter.call(new_metric, metric) if @filter && new_metric
+        expired
+      end
+    end
+
+    private
+
+    def get_time
+      ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+    end
+
+    def wrap_expire(method_name, &blk)
+      expire
+      @data.public_send(method_name, &blk)
+    end
+  end
+end

data/lib/prometheus_exporter/server/process_collector.rb

@@ -3,7 +3,8 @@
 module PrometheusExporter::Server
 
   class ProcessCollector < TypeCollector
-    MAX_PROCESS_METRIC_AGE = 60
+    MAX_METRIC_AGE = 60
+
     PROCESS_GAUGES = {
       heap_free_slots: "Free ruby heap slots.",
       heap_live_slots: "Used ruby heap slots.",
@@ -21,7 +22,10 @@ module PrometheusExporter::Server
     }
 
     def initialize
-      @process_metrics = []
+      @process_metrics = MetricsContainer.new(ttl: MAX_METRIC_AGE)
+      @process_metrics.filter = -> (new_metric, old_metric) do
+        new_metric["pid"] == old_metric["pid"] && new_metric["hostname"] == old_metric["hostname"]
+      end
     end
 
     def type
@@ -34,8 +38,8 @@ module PrometheusExporter::Server
       metrics = {}
 
       @process_metrics.map do |m|
-        metric_key = m["metric_labels"].merge("pid" => m["pid"])
-        metric_key.merge!(m["custom_labels"] || {})
+        metric_key = (m["metric_labels"] || {}).merge("pid" => m["pid"], "hostname" => m["hostname"])
+        metric_key.merge!(m["custom_labels"]) if m["custom_labels"]
 
         PROCESS_GAUGES.map do |k, help|
           k = k.to_s
@@ -58,15 +62,6 @@ module PrometheusExporter::Server
     end
 
     def collect(obj)
-      now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
-
-      obj["created_at"] = now
-
-      @process_metrics.delete_if do |current|
-        (obj["pid"] == current["pid"] && obj["hostname"] == current["hostname"]) ||
-          (current["created_at"] + MAX_PROCESS_METRIC_AGE < now)
-      end
-
       @process_metrics << obj
     end
   end

data/lib/prometheus_exporter/server/puma_collector.rb

@@ -4,17 +4,20 @@ module PrometheusExporter::Server
   class PumaCollector < TypeCollector
     MAX_PUMA_METRIC_AGE = 30
     PUMA_GAUGES = {
-      workers_total: "Number of puma workers.",
-      booted_workers_total: "Number of puma workers booted.",
-      old_workers_total: "Number of old puma workers.",
-      running_threads_total: "Number of puma threads currently running.",
-      request_backlog_total: "Number of requests waiting to be processed by a puma thread.",
-      thread_pool_capacity_total: "Number of puma threads available at current scale.",
-      max_threads_total: "Number of puma threads at available at max scale.",
+      workers: "Number of puma workers.",
+      booted_workers: "Number of puma workers booted.",
+      old_workers: "Number of old puma workers.",
+      running_threads: "Number of puma threads currently running.",
+      request_backlog: "Number of requests waiting to be processed by a puma thread.",
+      thread_pool_capacity: "Number of puma threads available at current scale.",
+      max_threads: "Number of puma threads at available at max scale.",
     }
 
     def initialize
-      @puma_metrics = []
+      @puma_metrics = MetricsContainer.new(ttl: MAX_PUMA_METRIC_AGE)
+      @puma_metrics.filter = -> (new_metric, old_metric) do
+        new_metric["pid"] == old_metric["pid"] && new_metric["hostname"] == old_metric["hostname"]
+      end
     end
 
     def type
@@ -34,6 +37,9 @@ module PrometheusExporter::Server
         if m["custom_labels"]
           labels.merge!(m["custom_labels"])
         end
+        if m["metric_labels"]
+          labels.merge!(m["metric_labels"])
+        end
 
         PUMA_GAUGES.map do |k, help|
           k = k.to_s
@@ -48,10 +54,6 @@ module PrometheusExporter::Server
     end
 
     def collect(obj)
-      now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
-
-      obj["created_at"] = now
-      @puma_metrics.delete_if { |m| m["created_at"] + MAX_PUMA_METRIC_AGE < now }
       @puma_metrics << obj
     end
   end

data/lib/prometheus_exporter/server/resque_collector.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module PrometheusExporter::Server
+  class ResqueCollector < TypeCollector
+    MAX_METRIC_AGE = 30
+    RESQUE_GAUGES = {
+      processed_jobs: "Total number of processed Resque jobs.",
+      failed_jobs: "Total number of failed Resque jobs.",
+      pending_jobs: "Total number of pending Resque jobs.",
+      queues: "Total number of Resque queues.",
+      workers: "Total number of Resque workers running.",
+      working: "Total number of Resque workers working."
+    }
+
+    def initialize
+      @resque_metrics = MetricsContainer.new(ttl: MAX_METRIC_AGE)
+      @gauges = {}
+    end
+
+    def type
+      "resque"
+    end
+
+    def metrics
+      return [] if resque_metrics.length == 0
+
+      resque_metrics.map do |metric|
+        labels = metric.fetch("custom_labels", {})
+
+        RESQUE_GAUGES.map do |name, help|
+          name = name.to_s
+          if value = metric[name]
+            gauge = gauges[name] ||= PrometheusExporter::Metric::Gauge.new("resque_#{name}", help)
+            gauge.observe(value, labels)
+          end
+        end
+      end
+
+      gauges.values
+    end
+
+    def collect(object)
+      @resque_metrics << object
+    end
+
+    private
+
+    attr_reader :resque_metrics, :gauges
+  end
+end

data/lib/prometheus_exporter/server/runner.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
-require 'prometheus_exporter/client'
-require_relative '../instrumentation/unicorn'
+require_relative '../client'
 
 module PrometheusExporter::Server
   class RunnerException < StandardError; end
@@ -17,6 +16,7 @@ module PrometheusExporter::Server
       @prefix = nil
       @auth = nil
       @realm = nil
+      @histogram = nil
 
       options.each do |k, v|
         send("#{k}=", v) if self.class.method_defined?("#{k}=")
@@ -27,6 +27,10 @@ module PrometheusExporter::Server
       PrometheusExporter::Metric::Base.default_prefix = prefix
       PrometheusExporter::Metric::Base.default_labels = label
 
+      if histogram
+        PrometheusExporter::Metric::Base.default_aggregation = PrometheusExporter::Metric::Histogram
+      end
+
       register_type_collectors
 
       unless collector.is_a?(PrometheusExporter::Server::CollectorBase)
@@ -34,6 +38,9 @@ module PrometheusExporter::Server
       end
 
       if unicorn_listen_address && unicorn_pid_file
+
+        require_relative '../instrumentation'
+
         local_client = PrometheusExporter::LocalClient.new(collector: collector)
         PrometheusExporter::Instrumentation::Unicorn.start(
           pid_file: unicorn_pid_file,
@@ -47,7 +54,7 @@ module PrometheusExporter::Server
     end
 
     attr_accessor :unicorn_listen_address, :unicorn_pid_file
-    attr_writer :prefix, :port, :bind, :collector_class, :type_collectors, :timeout, :verbose, :server_class, :label, :auth, :realm
+    attr_writer :prefix, :port, :bind, :collector_class, :type_collectors, :timeout, :verbose, :server_class, :label, :auth, :realm, :histogram
 
     def auth
       @auth || nil
@@ -98,6 +105,10 @@ module PrometheusExporter::Server
       @label ||= PrometheusExporter::DEFAULT_LABEL
     end
 
+    def histogram
+      @histogram || false
+    end
+
     private
 
     def register_type_collectors

data/lib/prometheus_exporter/server/sidekiq_collector.rb

@@ -52,7 +52,7 @@ module PrometheusExporter::Server
       if !@sidekiq_jobs_total
 
         @sidekiq_job_duration_seconds =
-        PrometheusExporter::Metric::Summary.new(
+        PrometheusExporter::Metric::Base.default_aggregation.new(
           "sidekiq_job_duration_seconds", "Total time spent in sidekiq jobs.")
 
         @sidekiq_jobs_total =

data/lib/prometheus_exporter/server/sidekiq_process_collector.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module PrometheusExporter::Server
+  class SidekiqProcessCollector < PrometheusExporter::Server::TypeCollector
+    MAX_METRIC_AGE = 60
+
+    SIDEKIQ_PROCESS_GAUGES = {
+      'busy' => 'Number of running jobs',
+      'concurrency' => 'Maximum concurrency',
+    }.freeze
+
+    attr_reader :sidekiq_metrics, :gauges
+
+    def initialize
+      @sidekiq_metrics = MetricsContainer.new(ttl: MAX_METRIC_AGE)
+      @gauges = {}
+    end
+
+    def type
+      'sidekiq_process'
+    end
+
+    def metrics
+      SIDEKIQ_PROCESS_GAUGES.each_key { |name| gauges[name]&.reset! }
+
+      sidekiq_metrics.map do |metric|
+        labels = metric.fetch('labels', {})
+        SIDEKIQ_PROCESS_GAUGES.map do |name, help|
+          if (value = metric[name])
+            gauge = gauges[name] ||= PrometheusExporter::Metric::Gauge.new("sidekiq_process_#{name}", help)
+            gauge.observe(value, labels)
+          end
+        end
+      end
+
+      gauges.values
+    end
+
+    def collect(object)
+      @sidekiq_metrics << object["process"]
+    end
+  end
+end

data/lib/prometheus_exporter/server/sidekiq_queue_collector.rb

@@ -1,17 +1,17 @@
 # frozen_string_literal: true
 module PrometheusExporter::Server
   class SidekiqQueueCollector < TypeCollector
-    MAX_SIDEKIQ_METRIC_AGE = 60
+    MAX_METRIC_AGE = 60
 
     SIDEKIQ_QUEUE_GAUGES = {
-      'backlog_total' => 'Size of the sidekiq queue.',
+      'backlog' => 'Size of the sidekiq queue.',
       'latency_seconds' => 'Latency of the sidekiq queue.',
     }.freeze
 
     attr_reader :sidekiq_metrics, :gauges
 
     def initialize
-      @sidekiq_metrics = []
+      @sidekiq_metrics = MetricsContainer.new(ttl: MAX_METRIC_AGE)
       @gauges = {}
     end
 
@@ -20,6 +20,8 @@ module PrometheusExporter::Server
     end
 
     def metrics
+      SIDEKIQ_QUEUE_GAUGES.each_key { |name| gauges[name]&.reset! }
+
       sidekiq_metrics.map do |metric|
         labels = metric.fetch("labels", {})
         SIDEKIQ_QUEUE_GAUGES.map do |name, help|
@@ -34,12 +36,9 @@ module PrometheusExporter::Server
     end
 
     def collect(object)
-      now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
       object['queues'].each do |queue|
-        queue["created_at"] = now
         queue["labels"].merge!(object['custom_labels']) if object['custom_labels']
-        sidekiq_metrics.delete_if { |metric| metric['created_at'] + MAX_SIDEKIQ_METRIC_AGE < now }
-        sidekiq_metrics << queue
+        @sidekiq_metrics << queue
       end
     end
   end

data/lib/prometheus_exporter/server/sidekiq_stats_collector.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module PrometheusExporter::Server
+  class SidekiqStatsCollector < TypeCollector
+    MAX_METRIC_AGE = 60
+
+    SIDEKIQ_STATS_GAUGES = {
+      'dead_size' => 'Size of dead the queue',
+      'enqueued' => 'Number of enqueued jobs',
+      'failed' => 'Number of failed jobs',
+      'processed' => 'Total number of processed jobs',
+      'processes_size' => 'Number of processes',
+      'retry_size' => 'Size of the retries queue',
+      'scheduled_size' => 'Size of the scheduled queue',
+      'workers_size' => 'Number of jobs actively being processed',
+    }.freeze
+
+    attr_reader :sidekiq_metrics, :gauges
+
+    def initialize
+      @sidekiq_metrics = MetricsContainer.new(ttl: MAX_METRIC_AGE)
+      @gauges = {}
+    end
+
+    def type
+      'sidekiq_stats'
+    end
+
+    def metrics
+      SIDEKIQ_STATS_GAUGES.each_key { |name| gauges[name]&.reset! }
+
+      sidekiq_metrics.map do |metric|
+        SIDEKIQ_STATS_GAUGES.map do |name, help|
+          if (value = metric['stats'][name])
+            gauge = gauges[name] ||= PrometheusExporter::Metric::Gauge.new("sidekiq_stats_#{name}", help)
+            gauge.observe(value)
+          end
+        end
+      end
+
+      gauges.values
+    end
+
+    def collect(object)
+      @sidekiq_metrics << object
+    end
+  end
+end

data/lib/prometheus_exporter/server/type_collector.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "prometheus_exporter/server/metrics_container"
+
 module PrometheusExporter::Server
   class TypeCollector
     def type

data/lib/prometheus_exporter/server/unicorn_collector.rb

@@ -2,47 +2,46 @@
 
 # custom type collector for prometheus_exporter for handling the metrics sent from
 # PrometheusExporter::Instrumentation::Unicorn
-class PrometheusExporter::Server::UnicornCollector < PrometheusExporter::Server::TypeCollector
-  MAX_UNICORN_METRIC_AGE = 60
-
-  UNICORN_GAUGES = {
-    workers_total: 'Number of unicorn workers.',
-    active_workers_total: 'Number of active unicorn workers',
-    request_backlog_total: 'Number of requests waiting to be processed by a unicorn worker.'
-  }.freeze
-
-  def initialize
-    @unicorn_metrics = []
-  end
+module PrometheusExporter::Server
+  class UnicornCollector < PrometheusExporter::Server::TypeCollector
+    MAX_METRIC_AGE = 60
+
+    UNICORN_GAUGES = {
+      workers: 'Number of unicorn workers.',
+      active_workers: 'Number of active unicorn workers',
+      request_backlog: 'Number of requests waiting to be processed by a unicorn worker.'
+    }.freeze
+
+    def initialize
+      @unicorn_metrics = MetricsContainer.new(ttl: MAX_METRIC_AGE)
+    end
 
-  def type
-    'unicorn'
-  end
+    def type
+      'unicorn'
+    end
 
-  def metrics
-    return [] if @unicorn_metrics.length.zero?
+    def metrics
+      return [] if @unicorn_metrics.length.zero?
 
-    metrics = {}
+      metrics = {}
 
-    @unicorn_metrics.map do |m|
-      labels = m["custom_labels"] || {}
+      @unicorn_metrics.map do |m|
+        labels = m["custom_labels"] || {}
 
-      UNICORN_GAUGES.map do |k, help|
-        k = k.to_s
-        if (v = m[k])
-          g = metrics[k] ||= PrometheusExporter::Metric::Gauge.new("unicorn_#{k}", help)
-          g.observe(v, labels)
+        UNICORN_GAUGES.map do |k, help|
+          k = k.to_s
+          if (v = m[k])
+            g = metrics[k] ||= PrometheusExporter::Metric::Gauge.new("unicorn_#{k}", help)
+            g.observe(v, labels)
+          end
         end
       end
-    end
 
-    metrics.values
-  end
+      metrics.values
+    end
 
-  def collect(obj)
-    now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
-    obj["created_at"] = now
-    @unicorn_metrics.delete_if { |m| m['created_at'] + MAX_UNICORN_METRIC_AGE < now }
-    @unicorn_metrics << obj
+    def collect(obj)
+      @unicorn_metrics << obj
+    end
   end
 end