prometheus_exporter 0.6.0 → 1.0.0

data/lib/prometheus_exporter/instrumentation/puma.rb

@@ -5,8 +5,8 @@ require "json"
 # collects stats from puma
 module PrometheusExporter::Instrumentation
   class Puma
-    def self.start(client: nil, frequency: 30)
-      puma_collector = new
+    def self.start(client: nil, frequency: 30, labels: {})
+      puma_collector = new(labels)
       client ||= PrometheusExporter::Client.default
       Thread.new do
         while true
@@ -14,7 +14,7 @@ module PrometheusExporter::Instrumentation
             metric = puma_collector.collect
             client.send_json metric
           rescue => e
-            STDERR.puts("Prometheus Exporter Failed To Collect Puma Stats #{e}")
+            client.logger.error("Prometheus Exporter Failed To Collect Puma Stats #{e}")
           ensure
             sleep frequency
           end
@@ -22,21 +22,33 @@ module PrometheusExporter::Instrumentation
       end
     end
 
+    def initialize(metric_labels = {})
+      @metric_labels = metric_labels
+    end
+
     def collect
-      metric = {}
-      metric[:type] = "puma"
+      metric = {
+        pid: pid,
+        type: "puma",
+        hostname: ::PrometheusExporter.hostname,
+        metric_labels: @metric_labels
+      }
       collect_puma_stats(metric)
       metric
     end
 
+    def pid
+      @pid = ::Process.pid
+    end
+
     def collect_puma_stats(metric)
       stats = JSON.parse(::Puma.stats)
 
       if stats.key?("workers")
         metric[:phase] = stats["phase"]
-        metric[:workers_total] = stats["workers"]
-        metric[:booted_workers_total] = stats["booted_workers"]
-        metric[:old_workers_total] = stats["old_workers"]
+        metric[:workers] = stats["workers"]
+        metric[:booted_workers] = stats["booted_workers"]
+        metric[:old_workers] = stats["old_workers"]
 
         stats["worker_status"].each do |worker|
           next if worker["last_status"].empty?
@@ -50,15 +62,15 @@ module PrometheusExporter::Instrumentation
     private
 
     def collect_worker_status(metric, status)
-      metric[:request_backlog_total] ||= 0
-      metric[:running_threads_total] ||= 0
-      metric[:thread_pool_capacity_total] ||= 0
-      metric[:max_threads_total] ||= 0
+      metric[:request_backlog] ||= 0
+      metric[:running_threads] ||= 0
+      metric[:thread_pool_capacity] ||= 0
+      metric[:max_threads] ||= 0
 
-      metric[:request_backlog_total] += status["backlog"]
-      metric[:running_threads_total] += status["running"]
-      metric[:thread_pool_capacity_total] += status["pool_capacity"]
-      metric[:max_threads_total] += status["max_threads"]
+      metric[:request_backlog] += status["backlog"]
+      metric[:running_threads] += status["running"]
+      metric[:thread_pool_capacity] += status["pool_capacity"]
+      metric[:max_threads] += status["max_threads"]
     end
   end
 end

data/lib/prometheus_exporter/instrumentation/resque.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+# collects stats from resque
+module PrometheusExporter::Instrumentation
+  class Resque
+    def self.start(client: nil, frequency: 30)
+      resque_collector = new
+      client ||= PrometheusExporter::Client.default
+      Thread.new do
+        while true
+          begin
+            client.send_json(resque_collector.collect)
+          rescue => e
+            client.logger.error("Prometheus Exporter Failed To Collect Resque Stats #{e}")
+          ensure
+            sleep frequency
+          end
+        end
+      end
+    end
+
+    def collect
+      metric = {}
+      metric[:type] = "resque"
+      collect_resque_stats(metric)
+      metric
+    end
+
+    def collect_resque_stats(metric)
+      info = ::Resque.info
+
+      metric[:processed_jobs] = info[:processed]
+      metric[:failed_jobs] = info[:failed]
+      metric[:pending_jobs] = info[:pending]
+      metric[:queues] = info[:queues]
+      metric[:worker] = info[:workers]
+      metric[:working] = info[:working]
+    end
+  end
+end

data/lib/prometheus_exporter/instrumentation/sidekiq_process.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module PrometheusExporter::Instrumentation
+  class SidekiqProcess
+    def self.start(client: nil, frequency: 30)
+      client ||= PrometheusExporter::Client.default
+      sidekiq_process_collector = new
+
+      Thread.new do
+        loop do
+          begin
+            client.send_json(sidekiq_process_collector.collect)
+          rescue StandardError => e
+            STDERR.puts("Prometheus Exporter Failed To Collect Sidekiq Processes metrics #{e}")
+          ensure
+            sleep frequency
+          end
+        end
+      end
+    end
+
+    def initialize
+      @pid = ::Process.pid
+      @hostname = Socket.gethostname
+    end
+
+    def collect
+      {
+        type: 'sidekiq_process',
+        process: collect_stats
+      }
+    end
+
+    def collect_stats
+      process = current_process
+      return {} unless process
+
+      {
+        busy: process['busy'],
+        concurrency: process['concurrency'],
+        labels: {
+          labels: process['labels'].sort.join(','),
+          queues: process['queues'].sort.join(','),
+          quiet: process['quiet'],
+          tag: process['tag'],
+          hostname: process['hostname'],
+          identity: process['identity'],
+        }
+      }
+    end
+
+    def current_process
+      ::Sidekiq::ProcessSet.new.find do |sp|
+        sp['hostname'] == @hostname && sp['pid'] == @pid
+      end
+    end
+  end
+end

data/lib/prometheus_exporter/instrumentation/sidekiq_queue.rb

@@ -2,16 +2,16 @@
 
 module PrometheusExporter::Instrumentation
   class SidekiqQueue
-    def self.start(client: nil, frequency: 30)
+    def self.start(client: nil, frequency: 30, all_queues: false)
       client ||= PrometheusExporter::Client.default
-      sidekiq_queue_collector = new
+      sidekiq_queue_collector = new(all_queues: all_queues)
 
       Thread.new do
         loop do
           begin
             client.send_json(sidekiq_queue_collector.collect)
           rescue StandardError => e
-            STDERR.puts("Prometheus Exporter Failed To Collect Sidekiq Queue metrics #{e}")
+            client.logger.error("Prometheus Exporter Failed To Collect Sidekiq Queue metrics #{e}")
           ensure
             sleep frequency
           end
@@ -19,6 +19,12 @@ module PrometheusExporter::Instrumentation
       end
     end
 
+    def initialize(all_queues: false)
+      @all_queues = all_queues
+      @pid = ::Process.pid
+      @hostname = Socket.gethostname
+    end
+
     def collect
       {
         type: 'sidekiq_queue',
@@ -27,24 +33,32 @@ module PrometheusExporter::Instrumentation
     end
 
     def collect_queue_stats
-      hostname = Socket.gethostname
-      pid = ::Process.pid
-      ps = ::Sidekiq::ProcessSet.new
+      sidekiq_queues = ::Sidekiq::Queue.all
 
-      process = ps.find do |sp|
-        sp['hostname'] == hostname && sp['pid'] == pid
+      unless @all_queues
+        queues = collect_current_process_queues
+        sidekiq_queues.select! { |sidekiq_queue| queues.include?(sidekiq_queue.name) }
       end
 
-      queues = process.nil? ? [] : process['queues']
-
-      ::Sidekiq::Queue.all.map do |queue|
-        next unless queues.include? queue.name
+      sidekiq_queues.map do |queue|
         {
-          backlog_total: queue.size,
+          backlog: queue.size,
           latency_seconds: queue.latency.to_i,
           labels: { queue: queue.name }
         }
       end.compact
     end
+
+    private
+
+    def collect_current_process_queues
+      ps = ::Sidekiq::ProcessSet.new
+
+      process = ps.find do |sp|
+        sp['hostname'] == @hostname && sp['pid'] == @pid
+      end
+
+      process.nil? ? [] : process['queues']
+    end
   end
 end

data/lib/prometheus_exporter/instrumentation/unicorn.rb

@@ -18,7 +18,7 @@ module PrometheusExporter::Instrumentation
             metric = unicorn_collector.collect
             client.send_json metric
           rescue StandardError => e
-            STDERR.puts("Prometheus Exporter Failed To Collect Unicorn Stats #{e}")
+            client.logger.error("Prometheus Exporter Failed To Collect Unicorn Stats #{e}")
           ensure
             sleep frequency
           end
@@ -42,9 +42,9 @@ module PrometheusExporter::Instrumentation
     def collect_unicorn_stats(metric)
       stats = listener_address_stats
 
-      metric[:active_workers_total] = stats.active
-      metric[:request_backlog_total] = stats.queued
-      metric[:workers_total] = worker_process_count
+      metric[:active_workers] = stats.active
+      metric[:request_backlog] = stats.queued
+      metric[:workers] = worker_process_count
     end
 
     private

data/lib/prometheus_exporter/instrumentation.rb

@@ -5,9 +5,11 @@ require_relative "instrumentation/process"
 require_relative "instrumentation/method_profiler"
 require_relative "instrumentation/sidekiq"
 require_relative "instrumentation/sidekiq_queue"
+require_relative "instrumentation/sidekiq_process"
 require_relative "instrumentation/delayed_job"
 require_relative "instrumentation/puma"
 require_relative "instrumentation/hutch"
 require_relative "instrumentation/unicorn"
 require_relative "instrumentation/active_record"
 require_relative "instrumentation/shoryuken"
+require_relative "instrumentation/resque"

data/lib/prometheus_exporter/metric/base.rb

@@ -5,6 +5,7 @@ module PrometheusExporter::Metric
 
     @default_prefix = nil if !defined?(@default_prefix)
     @default_labels = nil if !defined?(@default_labels)
+    @default_aggregation = nil if !defined?(@default_aggregation)
 
     # prefix applied to all metrics
     def self.default_prefix=(name)
@@ -23,6 +24,14 @@ module PrometheusExporter::Metric
       @default_labels || {}
     end
 
+    def self.default_aggregation=(aggregation)
+      @default_aggregation = aggregation
+    end
+
+    def self.default_aggregation
+      @default_aggregation ||= Summary
+    end
+
     attr_accessor :help, :name, :data
 
     def initialize(name, help)

data/lib/prometheus_exporter/metric/gauge.rb

@@ -5,6 +5,10 @@ module PrometheusExporter::Metric
     attr_reader :data
 
     def initialize(name, help)
+      if name.end_with?("_total")
+        raise ArgumentError, "The metric name of gauge must not have _total suffix. Given: #{name}"
+      end
+
       super
       reset!
     end

data/lib/prometheus_exporter/middleware.rb

@@ -36,21 +36,12 @@ class PrometheusExporter::Middleware
 
     result
   ensure
-    status = (result && result[0]) || -1
-    params = env["action_dispatch.request.parameters"]
-    action, controller = nil
-    if params
-      action = params["action"]
-      controller = params["controller"]
-    end
 
     obj = {
       type: "web",
       timings: info,
       queue_time: queue_time,
-      action: action,
-      controller: controller,
-      status: status
+      default_labels: default_labels(env, result)
     }
     labels = custom_labels(env)
     if labels
@@ -60,6 +51,22 @@ class PrometheusExporter::Middleware
     @client.send_json(obj)
   end
 
+  def default_labels(env, result)
+    status = (result && result[0]) || -1
+    params = env["action_dispatch.request.parameters"]
+    action = controller = nil
+    if params
+      action = params["action"]
+      controller = params["controller"]
+    end
+
+    {
+      action: action || "other",
+      controller: controller || "other",
+      status: status
+    }
+  end
+
   # allows subclasses to add custom labels based on env
   def custom_labels(env)
     nil
@@ -83,19 +90,24 @@ class PrometheusExporter::Middleware
     Process.clock_gettime(Process::CLOCK_REALTIME)
   end
 
-  # get the content of the x-queue-start or x-request-start header
+  # determine queue start from well-known trace headers
   def queue_start(env)
+
+    # get the content of the x-queue-start or x-request-start header
     value = env['HTTP_X_REQUEST_START'] || env['HTTP_X_QUEUE_START']
     unless value.nil? || value == ''
-      convert_header_to_ms(value.to_s)
+      # nginx returns time as milliseconds with 3 decimal places
+      # apache returns time as microseconds without decimal places
+      # this method takes care to convert both into a proper second + fractions timestamp
+      value = value.to_s.gsub(/t=|\./, '')
+      return "#{value[0, 10]}.#{value[10, 13]}".to_f
     end
-  end
 
-  # nginx returns time as milliseconds with 3 decimal places
-  # apache returns time as microseconds without decimal places
-  # this method takes care to convert both into a proper second + fractions timestamp
-  def convert_header_to_ms(str)
-    str = str.gsub(/t=|\./, '')
-    "#{str[0, 10]}.#{str[10, 13]}".to_f
+    # get the content of the x-amzn-trace-id header
+    # see also: https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-request-tracing.html
+    value = env['HTTP_X_AMZN_TRACE_ID']
+    value&.split('Root=')&.last&.split('-')&.fetch(1)&.to_i(16)
+
   end
+
 end

data/lib/prometheus_exporter/server/active_record_collector.rb

@@ -47,7 +47,8 @@ module PrometheusExporter::Server
       obj["created_at"] = now
 
       @active_record_metrics.delete_if do |current|
-        (obj["pid"] == current["pid"] && obj["hostname"] == current["hostname"]) ||
+        (obj["pid"] == current["pid"] && obj["hostname"] == current["hostname"] &&
+         obj["metric_labels"]["pool_name"] == current["metric_labels"]["pool_name"]) ||
           (current["created_at"] + MAX_ACTIVERECORD_METRIC_AGE < now)
       end
 

data/lib/prometheus_exporter/server/collector.rb

@@ -14,12 +14,14 @@ module PrometheusExporter::Server
       register_collector(ProcessCollector.new)
       register_collector(SidekiqCollector.new)
       register_collector(SidekiqQueueCollector.new)
+      register_collector(SidekiqProcessCollector.new)
       register_collector(DelayedJobCollector.new)
       register_collector(PumaCollector.new)
       register_collector(HutchCollector.new)
       register_collector(UnicornCollector.new)
       register_collector(ActiveRecordCollector.new)
       register_collector(ShoryukenCollector.new)
+      register_collector(ResqueCollector.new)
     end
 
     def register_collector(collector)

data/lib/prometheus_exporter/server/delayed_job_collector.rb

@@ -19,21 +19,21 @@ module PrometheusExporter::Server
     end
 
     def collect(obj)
-      default_labels = { job_name: obj['name'] }
-      custom_labels = obj['custom_labels']
-      labels = custom_labels.nil? ? default_labels : default_labels.merge(custom_labels)
+      custom_labels = obj['custom_labels'] || {}
+      gauge_labels = { queue_name: obj['queue_name'] }.merge(custom_labels)
+      counter_labels = gauge_labels.merge(job_name: obj['name'])
 
       ensure_delayed_job_metrics
-      @delayed_job_duration_seconds.observe(obj["duration"], labels)
-      @delayed_jobs_total.observe(1, labels)
-      @delayed_failed_jobs_total.observe(1, labels) if !obj["success"]
-      @delayed_jobs_max_attempts_reached_total.observe(1) if obj["attempts"] >= obj["max_attempts"]
-      @delayed_job_duration_seconds_summary.observe(obj["duration"])
-      @delayed_job_duration_seconds_summary.observe(obj["duration"], status: "success") if obj["success"]
-      @delayed_job_duration_seconds_summary.observe(obj["duration"], status: "failed")  if !obj["success"]
-      @delayed_job_attempts_summary.observe(obj["attempts"]) if obj["success"]
-      @delayed_jobs_enqueued.observe(obj["enqueued"])
-      @delayed_jobs_pending.observe(obj["pending"])
+      @delayed_job_duration_seconds.observe(obj["duration"], counter_labels)
+      @delayed_jobs_total.observe(1, counter_labels)
+      @delayed_failed_jobs_total.observe(1, counter_labels) if !obj["success"]
+      @delayed_jobs_max_attempts_reached_total.observe(1, counter_labels) if obj["attempts"] >= obj["max_attempts"]
+      @delayed_job_duration_seconds_summary.observe(obj["duration"], counter_labels)
+      @delayed_job_duration_seconds_summary.observe(obj["duration"], counter_labels.merge(status: "success")) if obj["success"]
+      @delayed_job_duration_seconds_summary.observe(obj["duration"], counter_labels.merge(status: "failed"))  if !obj["success"]
+      @delayed_job_attempts_summary.observe(obj["attempts"], counter_labels) if obj["success"]
+      @delayed_jobs_enqueued.observe(obj["enqueued"], gauge_labels)
+      @delayed_jobs_pending.observe(obj["pending"], gauge_labels)
     end
 
     def metrics
@@ -76,12 +76,12 @@ module PrometheusExporter::Server
                 "delayed_jobs_max_attempts_reached_total", "Total number of delayed jobs that reached max attempts.")
 
         @delayed_job_duration_seconds_summary =
-            PrometheusExporter::Metric::Summary.new("delayed_job_duration_seconds_summary",
-                                                    "Summary of the time it takes jobs to execute.")
+            PrometheusExporter::Metric::Base.default_aggregation.new("delayed_job_duration_seconds_summary",
+                                                                     "Summary of the time it takes jobs to execute.")
 
         @delayed_job_attempts_summary =
-            PrometheusExporter::Metric::Summary.new("delayed_job_attempts_summary",
-                                                    "Summary of the amount of attempts it takes delayed jobs to succeed.")
+            PrometheusExporter::Metric::Base.default_aggregation.new("delayed_job_attempts_summary",
+                                                                     "Summary of the amount of attempts it takes delayed jobs to succeed.")
       end
     end
   end

data/lib/prometheus_exporter/server/puma_collector.rb

@@ -4,13 +4,13 @@ module PrometheusExporter::Server
   class PumaCollector < TypeCollector
     MAX_PUMA_METRIC_AGE = 30
     PUMA_GAUGES = {
-      workers_total: "Number of puma workers.",
-      booted_workers_total: "Number of puma workers booted.",
-      old_workers_total: "Number of old puma workers.",
-      running_threads_total: "Number of puma threads currently running.",
-      request_backlog_total: "Number of requests waiting to be processed by a puma thread.",
-      thread_pool_capacity_total: "Number of puma threads available at current scale.",
-      max_threads_total: "Number of puma threads at available at max scale.",
+      workers: "Number of puma workers.",
+      booted_workers: "Number of puma workers booted.",
+      old_workers: "Number of old puma workers.",
+      running_threads: "Number of puma threads currently running.",
+      request_backlog: "Number of requests waiting to be processed by a puma thread.",
+      thread_pool_capacity: "Number of puma threads available at current scale.",
+      max_threads: "Number of puma threads at available at max scale.",
     }
 
     def initialize
@@ -34,6 +34,9 @@ module PrometheusExporter::Server
         if m["custom_labels"]
           labels.merge!(m["custom_labels"])
         end
+        if m["metric_labels"]
+          labels.merge!(m["metric_labels"])
+        end
 
         PUMA_GAUGES.map do |k, help|
           k = k.to_s
@@ -51,7 +54,12 @@ module PrometheusExporter::Server
       now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
 
       obj["created_at"] = now
-      @puma_metrics.delete_if { |m| m["created_at"] + MAX_PUMA_METRIC_AGE < now }
+
+      @puma_metrics.delete_if do |current|
+        (obj["pid"] == current["pid"] && obj["hostname"] == current["hostname"]) ||
+          (current["created_at"] + MAX_PUMA_METRIC_AGE < now)
+      end
+
       @puma_metrics << obj
     end
   end

data/lib/prometheus_exporter/server/resque_collector.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module PrometheusExporter::Server
+  class ResqueCollector < TypeCollector
+    MAX_RESQUE_METRIC_AGE = 30
+    RESQUE_GAUGES = {
+      processed_jobs: "Total number of processed Resque jobs.",
+      failed_jobs: "Total number of failed Resque jobs.",
+      pending_jobs: "Total number of pending Resque jobs.",
+      queues: "Total number of Resque queues.",
+      workers: "Total number of Resque workers running.",
+      working: "Total number of Resque workers working."
+    }
+
+    def initialize
+      @resque_metrics = []
+      @gauges = {}
+    end
+
+    def type
+      "resque"
+    end
+
+    def metrics
+      return [] if resque_metrics.length == 0
+
+      resque_metrics.map do |metric|
+        labels = metric.fetch("custom_labels", {})
+
+        RESQUE_GAUGES.map do |name, help|
+          name = name.to_s
+          if value = metric[name]
+            gauge = gauges[name] ||= PrometheusExporter::Metric::Gauge.new("resque_#{name}", help)
+            gauge.observe(value, labels)
+          end
+        end
+      end
+
+      gauges.values
+    end
+
+    def collect(object)
+      now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+
+      object["created_at"] = now
+      resque_metrics.delete_if { |metric| metric["created_at"] + MAX_RESQUE_METRIC_AGE < now }
+      resque_metrics << object
+    end
+
+    private
+
+    attr_reader :resque_metrics, :gauges
+  end
+end

data/lib/prometheus_exporter/server/runner.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'prometheus_exporter/client'
+require_relative '../client'
 require_relative '../instrumentation/unicorn'
 
 module PrometheusExporter::Server
@@ -17,6 +17,7 @@ module PrometheusExporter::Server
       @prefix = nil
       @auth = nil
       @realm = nil
+      @histogram = nil
 
       options.each do |k, v|
         send("#{k}=", v) if self.class.method_defined?("#{k}=")
@@ -27,6 +28,10 @@ module PrometheusExporter::Server
       PrometheusExporter::Metric::Base.default_prefix = prefix
       PrometheusExporter::Metric::Base.default_labels = label
 
+      if histogram
+        PrometheusExporter::Metric::Base.default_aggregation = PrometheusExporter::Metric::Histogram
+      end
+
       register_type_collectors
 
       unless collector.is_a?(PrometheusExporter::Server::CollectorBase)
@@ -47,7 +52,7 @@ module PrometheusExporter::Server
     end
 
     attr_accessor :unicorn_listen_address, :unicorn_pid_file
-    attr_writer :prefix, :port, :bind, :collector_class, :type_collectors, :timeout, :verbose, :server_class, :label, :auth, :realm
+    attr_writer :prefix, :port, :bind, :collector_class, :type_collectors, :timeout, :verbose, :server_class, :label, :auth, :realm, :histogram
 
     def auth
       @auth || nil
@@ -98,6 +103,10 @@ module PrometheusExporter::Server
       @label ||= PrometheusExporter::DEFAULT_LABEL
     end
 
+    def histogram
+      @histogram || false
+    end
+
     private
 
     def register_type_collectors

data/lib/prometheus_exporter/server/sidekiq_collector.rb

@@ -52,7 +52,7 @@ module PrometheusExporter::Server
       if !@sidekiq_jobs_total
 
         @sidekiq_job_duration_seconds =
-        PrometheusExporter::Metric::Summary.new(
+        PrometheusExporter::Metric::Base.default_aggregation.new(
           "sidekiq_job_duration_seconds", "Total time spent in sidekiq jobs.")
 
         @sidekiq_jobs_total =