prometheus_exporter 0.7.0 → 2.3.0

data/lib/prometheus_exporter/server/hutch_collector.rb

@@ -14,8 +14,8 @@ module PrometheusExporter::Server
     end
 
     def collect(obj)
-      default_labels = { job_name: obj['name'] }
-      custom_labels = obj['custom_labels']
+      default_labels = { job_name: obj["name"] }
+      custom_labels = obj["custom_labels"]
       labels = custom_labels.nil? ? default_labels : default_labels.merge(custom_labels)
 
       ensure_hutch_metrics
@@ -36,15 +36,23 @@ module PrometheusExporter::Server
 
     def ensure_hutch_metrics
       if !@hutch_jobs_total
-
-        @hutch_job_duration_seconds = PrometheusExporter::Metric::Counter.new(
-          "hutch_job_duration_seconds", "Total time spent in hutch jobs.")
-
-        @hutch_jobs_total = PrometheusExporter::Metric::Counter.new(
-          "hutch_jobs_total", "Total number of hutch jobs executed.")
-
-        @hutch_failed_jobs_total = PrometheusExporter::Metric::Counter.new(
-          "hutch_failed_jobs_total", "Total number failed hutch jobs executed.")
+        @hutch_job_duration_seconds =
+          PrometheusExporter::Metric::Counter.new(
+            "hutch_job_duration_seconds",
+            "Total time spent in hutch jobs.",
+          )
+
+        @hutch_jobs_total =
+          PrometheusExporter::Metric::Counter.new(
+            "hutch_jobs_total",
+            "Total number of hutch jobs executed.",
+          )
+
+        @hutch_failed_jobs_total =
+          PrometheusExporter::Metric::Counter.new(
+            "hutch_failed_jobs_total",
+            "Total number failed hutch jobs executed.",
+          )
       end
     end
   end

data/lib/prometheus_exporter/server/metrics_container.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module PrometheusExporter::Server
+  class MetricsContainer
+    METRIC_MAX_AGE = 60
+    METRIC_EXPIRE_ATTR = "_expire_at"
+
+    attr_reader :data, :ttl
+    attr_accessor :filter
+
+    def initialize(ttl: METRIC_MAX_AGE, expire_attr: METRIC_EXPIRE_ATTR, filter: nil)
+      @data = []
+      @ttl = ttl
+      @expire_attr = expire_attr
+      @filter = filter
+    end
+
+    def <<(obj)
+      now = get_time
+      obj[@expire_attr] = now + @ttl
+
+      expire(time: now, new_metric: obj)
+
+      @data << obj
+      @data
+    end
+
+    def [](key)
+      @data.tap { expire }[key]
+    end
+
+    def size(&blk)
+      wrap_expire(:size, &blk)
+    end
+    alias_method :length, :size
+
+    def map(&blk)
+      wrap_expire(:map, &blk)
+    end
+
+    def each(&blk)
+      wrap_expire(:each, &blk)
+    end
+
+    def expire(time: nil, new_metric: nil)
+      time ||= get_time
+
+      @data.delete_if do |metric|
+        expired = metric[@expire_attr] < time
+        expired ||= filter.call(new_metric, metric) if @filter && new_metric
+        expired
+      end
+    end
+
+    private
+
+    def get_time
+      ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+    end
+
+    def wrap_expire(method_name, &blk)
+      expire
+      @data.public_send(method_name, &blk)
+    end
+  end
+end

data/lib/prometheus_exporter/server/process_collector.rb

@@ -1,9 +1,9 @@
 # frozen_string_literal: true
 
 module PrometheusExporter::Server
-
   class ProcessCollector < TypeCollector
-    MAX_PROCESS_METRIC_AGE = 60
+    MAX_METRIC_AGE = 60
+
     PROCESS_GAUGES = {
       heap_free_slots: "Free ruby heap slots.",
       heap_live_slots: "Used ruby heap slots.",
@@ -12,6 +12,12 @@ module PrometheusExporter::Server
       v8_physical_size: "Physical size consumed by V8 heaps.",
       v8_heap_count: "Number of V8 contexts running.",
       rss: "Total RSS used by process.",
+      malloc_increase_bytes_limit:
+        "Limit before Ruby triggers a GC against current objects (bytes).",
+      oldmalloc_increase_bytes_limit:
+        "Limit before Ruby triggers a major GC against old objects (bytes).",
+      marking_time: "Time spent in GC marking.",
+      sweeping_time: "Time spent in GC sweeping.",
     }
 
     PROCESS_COUNTERS = {
@@ -21,7 +27,10 @@ module PrometheusExporter::Server
     }
 
     def initialize
-      @process_metrics = []
+      @process_metrics = MetricsContainer.new(ttl: MAX_METRIC_AGE)
+      @process_metrics.filter = ->(new_metric, old_metric) do
+        new_metric["pid"] == old_metric["pid"] && new_metric["hostname"] == old_metric["hostname"]
+      end
     end
 
     def type
@@ -34,8 +43,9 @@ module PrometheusExporter::Server
       metrics = {}
 
       @process_metrics.map do |m|
-        metric_key = m["metric_labels"].merge("pid" => m["pid"])
-        metric_key.merge!(m["custom_labels"] || {})
+        metric_key =
+          (m["metric_labels"] || {}).merge("pid" => m["pid"], "hostname" => m["hostname"])
+        metric_key.merge!(m["custom_labels"]) if m["custom_labels"]
 
         PROCESS_GAUGES.map do |k, help|
           k = k.to_s
@@ -58,15 +68,6 @@ module PrometheusExporter::Server
     end
 
     def collect(obj)
-      now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
-
-      obj["created_at"] = now
-
-      @process_metrics.delete_if do |current|
-        (obj["pid"] == current["pid"] && obj["hostname"] == current["hostname"]) ||
-          (current["created_at"] + MAX_PROCESS_METRIC_AGE < now)
-      end
-
       @process_metrics << obj
     end
   end

data/lib/prometheus_exporter/server/puma_collector.rb

@@ -4,17 +4,27 @@ module PrometheusExporter::Server
   class PumaCollector < TypeCollector
     MAX_PUMA_METRIC_AGE = 30
     PUMA_GAUGES = {
-      workers_total: "Number of puma workers.",
-      booted_workers_total: "Number of puma workers booted.",
-      old_workers_total: "Number of old puma workers.",
-      running_threads_total: "Number of puma threads currently running.",
-      request_backlog_total: "Number of requests waiting to be processed by a puma thread.",
-      thread_pool_capacity_total: "Number of puma threads available at current scale.",
-      max_threads_total: "Number of puma threads at available at max scale.",
+      workers: "Number of puma workers.",
+      booted_workers: "Number of puma workers booted.",
+      old_workers: "Number of old puma workers.",
+      running_threads: "Number of puma threads currently running.",
+      request_backlog: "Number of requests waiting to be processed by a puma thread.",
+      thread_pool_capacity: "Number of puma threads available at current scale.",
+      max_threads: "Number of puma threads at available at max scale.",
     }
 
+    if defined?(::Puma::Const) &&
+         Gem::Version.new(::Puma::Const::VERSION) >= Gem::Version.new("6.6.0")
+      PUMA_GAUGES[
+        :busy_threads
+      ] = "Wholistic stat reflecting the overall current state of work to be done and the capacity to do it"
+    end
+
     def initialize
-      @puma_metrics = []
+      @puma_metrics = MetricsContainer.new(ttl: MAX_PUMA_METRIC_AGE)
+      @puma_metrics.filter = ->(new_metric, old_metric) do
+        new_metric["pid"] == old_metric["pid"] && new_metric["hostname"] == old_metric["hostname"]
+      end
     end
 
     def type
@@ -28,12 +38,9 @@ module PrometheusExporter::Server
 
       @puma_metrics.map do |m|
         labels = {}
-        if m["phase"]
-          labels.merge!(phase: m["phase"])
-        end
-        if m["custom_labels"]
-          labels.merge!(m["custom_labels"])
-        end
+        labels.merge!(phase: m["phase"]) if m["phase"]
+        labels.merge!(m["custom_labels"]) if m["custom_labels"]
+        labels.merge!(m["metric_labels"]) if m["metric_labels"]
 
         PUMA_GAUGES.map do |k, help|
           k = k.to_s
@@ -48,10 +55,6 @@ module PrometheusExporter::Server
     end
 
     def collect(obj)
-      now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
-
-      obj["created_at"] = now
-      @puma_metrics.delete_if { |m| m["created_at"] + MAX_PUMA_METRIC_AGE < now }
       @puma_metrics << obj
     end
   end

data/lib/prometheus_exporter/server/resque_collector.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module PrometheusExporter::Server
+  class ResqueCollector < TypeCollector
+    MAX_METRIC_AGE = 30
+    RESQUE_GAUGES = {
+      processed_jobs: "Total number of processed Resque jobs.",
+      failed_jobs: "Total number of failed Resque jobs.",
+      pending_jobs: "Total number of pending Resque jobs.",
+      queues: "Total number of Resque queues.",
+      workers: "Total number of Resque workers running.",
+      working: "Total number of Resque workers working.",
+    }
+
+    def initialize
+      @resque_metrics = MetricsContainer.new(ttl: MAX_METRIC_AGE)
+      @gauges = {}
+    end
+
+    def type
+      "resque"
+    end
+
+    def metrics
+      return [] if resque_metrics.length == 0
+
+      resque_metrics.map do |metric|
+        labels = metric.fetch("custom_labels", {})
+
+        RESQUE_GAUGES.map do |name, help|
+          name = name.to_s
+          if value = metric[name]
+            gauge = gauges[name] ||= PrometheusExporter::Metric::Gauge.new("resque_#{name}", help)
+            gauge.observe(value, labels)
+          end
+        end
+      end
+
+      gauges.values
+    end
+
+    def collect(object)
+      @resque_metrics << object
+    end
+
+    private
+
+    attr_reader :resque_metrics, :gauges
+  end
+end

data/lib/prometheus_exporter/server/runner.rb

@@ -1,11 +1,13 @@
 # frozen_string_literal: true
 
-require 'prometheus_exporter/client'
-require_relative '../instrumentation/unicorn'
+require_relative "../client"
 
 module PrometheusExporter::Server
-  class RunnerException < StandardError; end
-  class WrongInheritance < RunnerException; end
+  class RunnerException < StandardError
+  end
+
+  class WrongInheritance < RunnerException
+  end
 
   class Runner
     def initialize(options = {})
@@ -17,37 +19,67 @@ module PrometheusExporter::Server
       @prefix = nil
       @auth = nil
       @realm = nil
+      @histogram = nil
 
-      options.each do |k, v|
-        send("#{k}=", v) if self.class.method_defined?("#{k}=")
-      end
+      options.each { |k, v| send("#{k}=", v) if self.class.method_defined?("#{k}=") }
     end
 
     def start
       PrometheusExporter::Metric::Base.default_prefix = prefix
       PrometheusExporter::Metric::Base.default_labels = label
 
+      if histogram
+        PrometheusExporter::Metric::Base.default_aggregation = PrometheusExporter::Metric::Histogram
+      end
+
       register_type_collectors
 
       unless collector.is_a?(PrometheusExporter::Server::CollectorBase)
-        raise WrongInheritance, 'Collector class must be inherited from PrometheusExporter::Server::CollectorBase'
+        raise WrongInheritance,
+              "Collector class must be inherited from PrometheusExporter::Server::CollectorBase"
       end
 
       if unicorn_listen_address && unicorn_pid_file
+        require_relative "../instrumentation"
+
         local_client = PrometheusExporter::LocalClient.new(collector: collector)
         PrometheusExporter::Instrumentation::Unicorn.start(
           pid_file: unicorn_pid_file,
           listener_address: unicorn_listen_address,
-          client: local_client
+          client: local_client,
         )
       end
 
-      server = server_class.new(port: port, bind: bind, collector: collector, timeout: timeout, verbose: verbose, auth: auth, realm: realm)
-      server.start
+      @server =
+        server_class.new(
+          port: port,
+          bind: bind,
+          collector: collector,
+          timeout: timeout,
+          verbose: verbose,
+          auth: auth,
+          realm: realm,
+        )
+      @server.start
+    end
+
+    def stop
+      @server.stop
     end
 
     attr_accessor :unicorn_listen_address, :unicorn_pid_file
-    attr_writer :prefix, :port, :bind, :collector_class, :type_collectors, :timeout, :verbose, :server_class, :label, :auth, :realm
+    attr_writer :prefix,
+                :port,
+                :bind,
+                :collector_class,
+                :type_collectors,
+                :timeout,
+                :verbose,
+                :server_class,
+                :label,
+                :auth,
+                :realm,
+                :histogram
 
     def auth
       @auth || nil
@@ -82,7 +114,7 @@ module PrometheusExporter::Server
     end
 
     def verbose
-      return @verbose if defined? @verbose
+      return @verbose if defined?(@verbose)
       false
     end
 
@@ -98,6 +130,10 @@ module PrometheusExporter::Server
       @label ||= PrometheusExporter::DEFAULT_LABEL
     end
 
+    def histogram
+      @histogram || false
+    end
+
     private
 
     def register_type_collectors

data/lib/prometheus_exporter/server/shoryuken_collector.rb

@@ -2,7 +2,6 @@
 
 module PrometheusExporter::Server
   class ShoryukenCollector < TypeCollector
-
     def initialize
       @shoryuken_jobs_total = nil
       @shoryuken_job_duration_seconds = nil
@@ -16,8 +15,8 @@ module PrometheusExporter::Server
     end
 
     def collect(obj)
-      default_labels = { job_name: obj['name'] , queue_name: obj['queue'] }
-      custom_labels = obj['custom_labels']
+      default_labels = { job_name: obj["name"], queue_name: obj["queue"] }
+      custom_labels = obj["custom_labels"]
       labels = custom_labels.nil? ? default_labels : default_labels.merge(custom_labels)
 
       ensure_shoryuken_metrics
@@ -30,10 +29,10 @@ module PrometheusExporter::Server
     def metrics
       if @shoryuken_jobs_total
         [
-            @shoryuken_job_duration_seconds,
-            @shoryuken_jobs_total,
-            @shoryuken_restarted_jobs_total,
-            @shoryuken_failed_jobs_total,
+          @shoryuken_job_duration_seconds,
+          @shoryuken_jobs_total,
+          @shoryuken_restarted_jobs_total,
+          @shoryuken_failed_jobs_total,
         ]
       else
         []
@@ -44,23 +43,29 @@ module PrometheusExporter::Server
 
     def ensure_shoryuken_metrics
       if !@shoryuken_jobs_total
-
         @shoryuken_job_duration_seconds =
-            PrometheusExporter::Metric::Counter.new(
-                "shoryuken_job_duration_seconds", "Total time spent in shoryuken jobs.")
+          PrometheusExporter::Metric::Counter.new(
+            "shoryuken_job_duration_seconds",
+            "Total time spent in shoryuken jobs.",
+          )
 
         @shoryuken_jobs_total =
-            PrometheusExporter::Metric::Counter.new(
-                "shoryuken_jobs_total", "Total number of shoryuken jobs executed.")
+          PrometheusExporter::Metric::Counter.new(
+            "shoryuken_jobs_total",
+            "Total number of shoryuken jobs executed.",
+          )
 
         @shoryuken_restarted_jobs_total =
-            PrometheusExporter::Metric::Counter.new(
-                "shoryuken_restarted_jobs_total", "Total number of shoryuken jobs that we restarted because of a shoryuken shutdown.")
+          PrometheusExporter::Metric::Counter.new(
+            "shoryuken_restarted_jobs_total",
+            "Total number of shoryuken jobs that we restarted because of a shoryuken shutdown.",
+          )
 
         @shoryuken_failed_jobs_total =
-            PrometheusExporter::Metric::Counter.new(
-                "shoryuken_failed_jobs_total", "Total number of failed shoryuken jobs.")
-
+          PrometheusExporter::Metric::Counter.new(
+            "shoryuken_failed_jobs_total",
+            "Total number of failed shoryuken jobs.",
+          )
       end
     end
   end

data/lib/prometheus_exporter/server/sidekiq_collector.rb

@@ -2,7 +2,6 @@
 
 module PrometheusExporter::Server
   class SidekiqCollector < TypeCollector
-
     def initialize
       @sidekiq_jobs_total = nil
       @sidekiq_job_duration_seconds = nil
@@ -17,8 +16,8 @@ module PrometheusExporter::Server
     end
 
     def collect(obj)
-      default_labels = { job_name: obj['name'], queue: obj['queue'] }
-      custom_labels = obj['custom_labels']
+      default_labels = { job_name: obj["name"], queue: obj["queue"] }
+      custom_labels = obj["custom_labels"]
       labels = custom_labels.nil? ? default_labels : default_labels.merge(custom_labels)
 
       ensure_sidekiq_metrics
@@ -50,26 +49,35 @@ module PrometheusExporter::Server
 
     def ensure_sidekiq_metrics
       if !@sidekiq_jobs_total
-
         @sidekiq_job_duration_seconds =
-        PrometheusExporter::Metric::Summary.new(
-          "sidekiq_job_duration_seconds", "Total time spent in sidekiq jobs.")
+          PrometheusExporter::Metric::Base.default_aggregation.new(
+            "sidekiq_job_duration_seconds",
+            "Total time spent in sidekiq jobs.",
+          )
 
         @sidekiq_jobs_total =
-        PrometheusExporter::Metric::Counter.new(
-          "sidekiq_jobs_total", "Total number of sidekiq jobs executed.")
+          PrometheusExporter::Metric::Counter.new(
+            "sidekiq_jobs_total",
+            "Total number of sidekiq jobs executed.",
+          )
 
         @sidekiq_restarted_jobs_total =
-        PrometheusExporter::Metric::Counter.new(
-          "sidekiq_restarted_jobs_total", "Total number of sidekiq jobs that we restarted because of a sidekiq shutdown.")
+          PrometheusExporter::Metric::Counter.new(
+            "sidekiq_restarted_jobs_total",
+            "Total number of sidekiq jobs that we restarted because of a sidekiq shutdown.",
+          )
 
         @sidekiq_failed_jobs_total =
-        PrometheusExporter::Metric::Counter.new(
-          "sidekiq_failed_jobs_total", "Total number of failed sidekiq jobs.")
+          PrometheusExporter::Metric::Counter.new(
+            "sidekiq_failed_jobs_total",
+            "Total number of failed sidekiq jobs.",
+          )
 
         @sidekiq_dead_jobs_total =
-        PrometheusExporter::Metric::Counter.new(
-          "sidekiq_dead_jobs_total", "Total number of dead sidekiq jobs.")
+          PrometheusExporter::Metric::Counter.new(
+            "sidekiq_dead_jobs_total",
+            "Total number of dead sidekiq jobs.",
+          )
       end
     end
   end

data/lib/prometheus_exporter/server/sidekiq_process_collector.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module PrometheusExporter::Server
+  class SidekiqProcessCollector < PrometheusExporter::Server::TypeCollector
+    MAX_METRIC_AGE = 60
+
+    SIDEKIQ_PROCESS_GAUGES = {
+      "busy" => "Number of running jobs",
+      "concurrency" => "Maximum concurrency",
+    }.freeze
+
+    attr_reader :sidekiq_metrics, :gauges
+
+    def initialize
+      @sidekiq_metrics = MetricsContainer.new(ttl: MAX_METRIC_AGE)
+      @gauges = {}
+    end
+
+    def type
+      "sidekiq_process"
+    end
+
+    def metrics
+      SIDEKIQ_PROCESS_GAUGES.each_key { |name| gauges[name]&.reset! }
+
+      sidekiq_metrics.map do |metric|
+        labels = metric.fetch("labels", {})
+        SIDEKIQ_PROCESS_GAUGES.map do |name, help|
+          if (value = metric[name])
+            gauge =
+              gauges[name] ||= PrometheusExporter::Metric::Gauge.new(
+                "sidekiq_process_#{name}",
+                help,
+              )
+            gauge.observe(value, labels)
+          end
+        end
+      end
+
+      gauges.values
+    end
+
+    def collect(object)
+      @sidekiq_metrics << object["process"]
+    end
+  end
+end

data/lib/prometheus_exporter/server/sidekiq_queue_collector.rb

@@ -1,30 +1,33 @@
 # frozen_string_literal: true
 module PrometheusExporter::Server
   class SidekiqQueueCollector < TypeCollector
-    MAX_SIDEKIQ_METRIC_AGE = 60
+    MAX_METRIC_AGE = 60
 
     SIDEKIQ_QUEUE_GAUGES = {
-      'backlog_total' => 'Size of the sidekiq queue.',
-      'latency_seconds' => 'Latency of the sidekiq queue.',
+      "backlog" => "Size of the sidekiq queue.",
+      "latency_seconds" => "Latency of the sidekiq queue.",
     }.freeze
 
     attr_reader :sidekiq_metrics, :gauges
 
     def initialize
-      @sidekiq_metrics = []
+      @sidekiq_metrics = MetricsContainer.new(ttl: MAX_METRIC_AGE)
       @gauges = {}
     end
 
     def type
-      'sidekiq_queue'
+      "sidekiq_queue"
     end
 
     def metrics
+      SIDEKIQ_QUEUE_GAUGES.each_key { |name| gauges[name]&.reset! }
+
       sidekiq_metrics.map do |metric|
         labels = metric.fetch("labels", {})
         SIDEKIQ_QUEUE_GAUGES.map do |name, help|
           if (value = metric[name])
-            gauge = gauges[name] ||= PrometheusExporter::Metric::Gauge.new("sidekiq_queue_#{name}", help)
+            gauge =
+              gauges[name] ||= PrometheusExporter::Metric::Gauge.new("sidekiq_queue_#{name}", help)
             gauge.observe(value, labels)
           end
         end
@@ -34,12 +37,9 @@ module PrometheusExporter::Server
     end
 
     def collect(object)
-      now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
-      object['queues'].each do |queue|
-        queue["created_at"] = now
-        queue["labels"].merge!(object['custom_labels']) if object['custom_labels']
-        sidekiq_metrics.delete_if { |metric| metric['created_at'] + MAX_SIDEKIQ_METRIC_AGE < now }
-        sidekiq_metrics << queue
+      object["queues"].each do |queue|
+        queue["labels"].merge!(object["custom_labels"]) if object["custom_labels"]
+        @sidekiq_metrics << queue
       end
     end
   end