prometheus_exporter 0.5.1 → 0.8.0

data/lib/prometheus_exporter/metric/base.rb

@@ -2,6 +2,10 @@
 
 module PrometheusExporter::Metric
   class Base
+
+    @default_prefix = nil if !defined?(@default_prefix)
+    @default_labels = nil if !defined?(@default_labels)
+
     # prefix applied to all metrics
     def self.default_prefix=(name)
       @default_prefix = name

data/lib/prometheus_exporter/metric/counter.rb

@@ -27,6 +27,10 @@ module PrometheusExporter::Metric
       @data.dup
     end
 
+    def remove(labels)
+      @data.delete(labels)
+    end
+
     def observe(increment = 1, labels = {})
       @data[labels] ||= 0
       @data[labels] += increment

data/lib/prometheus_exporter/metric/gauge.rb

@@ -27,6 +27,10 @@ module PrometheusExporter::Metric
       @data.dup
     end
 
+    def remove(labels)
+      @data.delete(labels)
+    end
+
     def observe(value, labels = {})
       if value.nil?
         data.delete(labels)

data/lib/prometheus_exporter/metric/histogram.rb

@@ -27,6 +27,12 @@ module PrometheusExporter::Metric
       data
     end
 
+    def remove(labels)
+      @observations.delete(labels)
+      @counts.delete(labels)
+      @sums.delete(labels)
+    end
+
     def type
       "histogram"
     end

data/lib/prometheus_exporter/metric/summary.rb

@@ -32,6 +32,13 @@ module PrometheusExporter::Metric
       data
     end
 
+    def remove(labels)
+      @counts.delete(labels)
+      @sums.delete(labels)
+      @buffers[0].delete(labels)
+      @buffers[1].delete(labels)
+    end
+
     def type
       "summary"
     end

data/lib/prometheus_exporter/middleware.rb

@@ -36,22 +36,40 @@ class PrometheusExporter::Middleware
 
     result
   ensure
+
+    obj = {
+      type: "web",
+      timings: info,
+      queue_time: queue_time,
+      default_labels: default_labels(env, result)
+    }
+    labels = custom_labels(env)
+    if labels
+      obj = obj.merge(custom_labels: labels)
+    end
+
+    @client.send_json(obj)
+  end
+
+  def default_labels(env, result)
     status = (result && result[0]) || -1
     params = env["action_dispatch.request.parameters"]
-    action, controller = nil
+    action = controller = nil
     if params
       action = params["action"]
       controller = params["controller"]
     end
 
-    @client.send_json(
-      type: "web",
-      timings: info,
-      queue_time: queue_time,
-      action: action,
-      controller: controller,
+    {
+      action: action || "other",
+      controller: controller || "other",
       status: status
-    )
+    }
+  end
+
+  # allows subclasses to add custom labels based on env
+  def custom_labels(env)
+    nil
   end
 
   private
@@ -72,19 +90,24 @@ class PrometheusExporter::Middleware
     Process.clock_gettime(Process::CLOCK_REALTIME)
   end
 
-  # get the content of the x-queue-start or x-request-start header
+  # determine queue start from well-known trace headers
   def queue_start(env)
+
+    # get the content of the x-queue-start or x-request-start header
     value = env['HTTP_X_REQUEST_START'] || env['HTTP_X_QUEUE_START']
     unless value.nil? || value == ''
-      convert_header_to_ms(value.to_s)
+      # nginx returns time as milliseconds with 3 decimal places
+      # apache returns time as microseconds without decimal places
+      # this method takes care to convert both into a proper second + fractions timestamp
+      value = value.to_s.gsub(/t=|\./, '')
+      return "#{value[0, 10]}.#{value[10, 13]}".to_f
     end
-  end
 
-  # nginx returns time as milliseconds with 3 decimal places
-  # apache returns time as microseconds without decimal places
-  # this method takes care to convert both into a proper second + fractions timestamp
-  def convert_header_to_ms(str)
-    str = str.gsub(/t=|\./, '')
-    "#{str[0, 10]}.#{str[10, 13]}".to_f
+    # get the content of the x-amzn-trace-id header
+    # see also: https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-request-tracing.html
+    value = env['HTTP_X_AMZN_TRACE_ID']
+    value&.split('Root=')&.last&.split('-')&.fetch(1)&.to_i(16)
+
   end
+
 end

data/lib/prometheus_exporter/server.rb

@@ -5,6 +5,7 @@ require_relative "server/type_collector"
 require_relative "server/web_collector"
 require_relative "server/process_collector"
 require_relative "server/sidekiq_collector"
+require_relative "server/sidekiq_queue_collector"
 require_relative "server/delayed_job_collector"
 require_relative "server/collector_base"
 require_relative "server/collector"
@@ -15,3 +16,4 @@ require_relative "server/hutch_collector"
 require_relative "server/unicorn_collector"
 require_relative "server/active_record_collector"
 require_relative "server/shoryuken_collector"
+require_relative "server/resque_collector"

data/lib/prometheus_exporter/server/active_record_collector.rb

@@ -27,6 +27,7 @@ module PrometheusExporter::Server
 
       @active_record_metrics.map do |m|
         metric_key = (m["metric_labels"] || {}).merge("pid" => m["pid"])
+        metric_key.merge!(m["custom_labels"]) if m["custom_labels"]
 
         ACTIVE_RECORD_GAUGES.map do |k, help|
           k = k.to_s
@@ -46,7 +47,8 @@ module PrometheusExporter::Server
       obj["created_at"] = now
 
       @active_record_metrics.delete_if do |current|
-        (obj["pid"] == current["pid"] && obj["hostname"] == current["hostname"]) ||
+        (obj["pid"] == current["pid"] && obj["hostname"] == current["hostname"] &&
+         obj["metric_labels"]["pool_name"] == current["metric_labels"]["pool_name"]) ||
           (current["created_at"] + MAX_ACTIVERECORD_METRIC_AGE < now)
       end
 

data/lib/prometheus_exporter/server/collector.rb

@@ -13,12 +13,14 @@ module PrometheusExporter::Server
       register_collector(WebCollector.new)
       register_collector(ProcessCollector.new)
       register_collector(SidekiqCollector.new)
+      register_collector(SidekiqQueueCollector.new)
       register_collector(DelayedJobCollector.new)
       register_collector(PumaCollector.new)
       register_collector(HutchCollector.new)
       register_collector(UnicornCollector.new)
       register_collector(ActiveRecordCollector.new)
       register_collector(ShoryukenCollector.new)
+      register_collector(ResqueCollector.new)
     end
 
     def register_collector(collector)

data/lib/prometheus_exporter/server/delayed_job_collector.rb

@@ -2,27 +2,39 @@
 
 module PrometheusExporter::Server
   class DelayedJobCollector < TypeCollector
+    def initialize
+      @delayed_jobs_total = nil
+      @delayed_job_duration_seconds = nil
+      @delayed_jobs_total = nil
+      @delayed_failed_jobs_total = nil
+      @delayed_jobs_max_attempts_reached_total = nil
+      @delayed_job_duration_seconds_summary = nil
+      @delayed_job_attempts_summary = nil
+      @delayed_jobs_enqueued = nil
+      @delayed_jobs_pending = nil
+    end
 
     def type
       "delayed_job"
     end
 
     def collect(obj)
-      default_labels = { job_name: obj['name'] }
+      default_labels = { job_name: obj['name'], queue_name: obj['queue_name'] }
       custom_labels = obj['custom_labels']
+
       labels = custom_labels.nil? ? default_labels : default_labels.merge(custom_labels)
 
       ensure_delayed_job_metrics
       @delayed_job_duration_seconds.observe(obj["duration"], labels)
       @delayed_jobs_total.observe(1, labels)
       @delayed_failed_jobs_total.observe(1, labels) if !obj["success"]
-      @delayed_jobs_max_attempts_reached_total.observe(1) if obj["attempts"] >= obj["max_attempts"]
-      @delayed_job_duration_seconds_summary.observe(obj["duration"])
-      @delayed_job_duration_seconds_summary.observe(obj["duration"], status: "success") if obj["success"]
-      @delayed_job_duration_seconds_summary.observe(obj["duration"], status: "failed")  if !obj["success"]
-      @delayed_job_attempts_summary.observe(obj["attempts"]) if obj["success"]
-      @delayed_jobs_enqueued.observe(obj["enqueued"])
-      @delayed_jobs_pending.observe(obj["pending"])
+      @delayed_jobs_max_attempts_reached_total.observe(1, labels) if obj["attempts"] >= obj["max_attempts"]
+      @delayed_job_duration_seconds_summary.observe(obj["duration"], labels)
+      @delayed_job_duration_seconds_summary.observe(obj["duration"], labels.merge(status: "success")) if obj["success"]
+      @delayed_job_duration_seconds_summary.observe(obj["duration"], labels.merge(status: "failed"))  if !obj["success"]
+      @delayed_job_attempts_summary.observe(obj["attempts"], labels) if obj["success"]
+      @delayed_jobs_enqueued.observe(obj["enqueued"], labels)
+      @delayed_jobs_pending.observe(obj["pending"], labels)
     end
 
     def metrics

data/lib/prometheus_exporter/server/hutch_collector.rb

@@ -2,6 +2,12 @@
 
 module PrometheusExporter::Server
   class HutchCollector < TypeCollector
+    def initialize
+      @hutch_jobs_total = nil
+      @hutch_job_duration_seconds = nil
+      @hutch_jobs_total = nil
+      @hutch_failed_jobs_total = nil
+    end
 
     def type
       "hutch"

data/lib/prometheus_exporter/server/puma_collector.rb

@@ -34,6 +34,9 @@ module PrometheusExporter::Server
         if m["custom_labels"]
           labels.merge!(m["custom_labels"])
         end
+        if m["metric_labels"]
+          labels.merge!(m["metric_labels"])
+        end
 
         PUMA_GAUGES.map do |k, help|
           k = k.to_s
@@ -51,7 +54,12 @@ module PrometheusExporter::Server
       now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
 
       obj["created_at"] = now
-      @puma_metrics.delete_if { |m| m["created_at"] + MAX_PUMA_METRIC_AGE < now }
+
+      @puma_metrics.delete_if do |current|
+        (obj["pid"] == current["pid"] && obj["hostname"] == current["hostname"]) ||
+          (current["created_at"] + MAX_PUMA_METRIC_AGE < now)
+      end
+
       @puma_metrics << obj
     end
   end

data/lib/prometheus_exporter/server/resque_collector.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module PrometheusExporter::Server
+  class ResqueCollector < TypeCollector
+    MAX_RESQUE_METRIC_AGE = 30
+    RESQUE_GAUGES = {
+      processed_jobs_total: "Total number of processed Resque jobs.",
+      failed_jobs_total: "Total number of failed Resque jobs.",
+      pending_jobs_total: "Total number of pending Resque jobs.",
+      queues_total: "Total number of Resque queues.",
+      workers_total: "Total number of Resque workers running.",
+      working_total: "Total number of Resque workers working."
+    }
+
+    def initialize
+      @resque_metrics = []
+      @gauges = {}
+    end
+
+    def type
+      "resque"
+    end
+
+    def metrics
+      return [] if resque_metrics.length == 0
+
+      resque_metrics.map do |metric|
+        labels = metric.fetch("custom_labels", {})
+
+        RESQUE_GAUGES.map do |name, help|
+          name = name.to_s
+          if value = metric[name]
+            gauge = gauges[name] ||= PrometheusExporter::Metric::Gauge.new("resque_#{name}", help)
+            gauge.observe(value, labels)
+          end
+        end
+      end
+
+      gauges.values
+    end
+
+    def collect(object)
+      now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+
+      object["created_at"] = now
+      resque_metrics.delete_if { |metric| metric["created_at"] + MAX_RESQUE_METRIC_AGE < now }
+      resque_metrics << object
+    end
+
+    private
+
+    attr_reader :resque_metrics, :gauges
+  end
+end

data/lib/prometheus_exporter/server/runner.rb

@@ -9,6 +9,15 @@ module PrometheusExporter::Server
 
   class Runner
     def initialize(options = {})
+      @timeout = nil
+      @port = nil
+      @bind = nil
+      @collector_class = nil
+      @type_collectors = nil
+      @prefix = nil
+      @auth = nil
+      @realm = nil
+
       options.each do |k, v|
         send("#{k}=", v) if self.class.method_defined?("#{k}=")
       end
@@ -16,6 +25,7 @@ module PrometheusExporter::Server
 
     def start
       PrometheusExporter::Metric::Base.default_prefix = prefix
+      PrometheusExporter::Metric::Base.default_labels = label
 
       register_type_collectors
 
@@ -32,12 +42,20 @@ module PrometheusExporter::Server
         )
       end
 
-      server = server_class.new port: port, bind: bind, collector: collector, timeout: timeout, verbose: verbose
+      server = server_class.new(port: port, bind: bind, collector: collector, timeout: timeout, verbose: verbose, auth: auth, realm: realm)
       server.start
     end
 
     attr_accessor :unicorn_listen_address, :unicorn_pid_file
-    attr_writer :prefix, :port, :bind, :collector_class, :type_collectors, :timeout, :verbose, :server_class
+    attr_writer :prefix, :port, :bind, :collector_class, :type_collectors, :timeout, :verbose, :server_class, :label, :auth, :realm
+
+    def auth
+      @auth || nil
+    end
+
+    def realm
+      @realm || PrometheusExporter::DEFAULT_REALM
+    end
 
     def prefix
       @prefix || PrometheusExporter::DEFAULT_PREFIX
@@ -76,6 +94,10 @@ module PrometheusExporter::Server
       @_collector ||= collector_class.new
     end
 
+    def label
+      @label ||= PrometheusExporter::DEFAULT_LABEL
+    end
+
     private
 
     def register_type_collectors

data/lib/prometheus_exporter/server/shoryuken_collector.rb

@@ -3,6 +3,14 @@
 module PrometheusExporter::Server
   class ShoryukenCollector < TypeCollector
 
+    def initialize
+      @shoryuken_jobs_total = nil
+      @shoryuken_job_duration_seconds = nil
+      @shoryuken_jobs_total = nil
+      @shoryuken_restarted_jobs_total = nil
+      @shoryuken_failed_jobs_total = nil
+    end
+
     def type
       "shoryuken"
     end

data/lib/prometheus_exporter/server/sidekiq_collector.rb

@@ -3,12 +3,21 @@
 module PrometheusExporter::Server
   class SidekiqCollector < TypeCollector
 
+    def initialize
+      @sidekiq_jobs_total = nil
+      @sidekiq_job_duration_seconds = nil
+      @sidekiq_jobs_total = nil
+      @sidekiq_restarted_jobs_total = nil
+      @sidekiq_failed_jobs_total = nil
+      @sidekiq_dead_jobs_total = nil
+    end
+
     def type
       "sidekiq"
     end
 
     def collect(obj)
-      default_labels = { job_name: obj['name'] }
+      default_labels = { job_name: obj['name'], queue: obj['queue'] }
       custom_labels = obj['custom_labels']
       labels = custom_labels.nil? ? default_labels : default_labels.merge(custom_labels)
 
@@ -43,7 +52,7 @@ module PrometheusExporter::Server
       if !@sidekiq_jobs_total
 
         @sidekiq_job_duration_seconds =
-        PrometheusExporter::Metric::Counter.new(
+        PrometheusExporter::Metric::Summary.new(
           "sidekiq_job_duration_seconds", "Total time spent in sidekiq jobs.")
 
         @sidekiq_jobs_total =

data/lib/prometheus_exporter/server/sidekiq_queue_collector.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+module PrometheusExporter::Server
+  class SidekiqQueueCollector < TypeCollector
+    MAX_SIDEKIQ_METRIC_AGE = 60
+
+    SIDEKIQ_QUEUE_GAUGES = {
+      'backlog_total' => 'Size of the sidekiq queue.',
+      'latency_seconds' => 'Latency of the sidekiq queue.',
+    }.freeze
+
+    attr_reader :sidekiq_metrics, :gauges
+
+    def initialize
+      @sidekiq_metrics = []
+      @gauges = {}
+    end
+
+    def type
+      'sidekiq_queue'
+    end
+
+    def metrics
+      sidekiq_metrics.map do |metric|
+        labels = metric.fetch("labels", {})
+        SIDEKIQ_QUEUE_GAUGES.map do |name, help|
+          if (value = metric[name])
+            gauge = gauges[name] ||= PrometheusExporter::Metric::Gauge.new("sidekiq_queue_#{name}", help)
+            gauge.observe(value, labels)
+          end
+        end
+      end
+
+      gauges.values
+    end
+
+    def collect(object)
+      now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      object['queues'].each do |queue|
+        queue["created_at"] = now
+        queue["labels"].merge!(object['custom_labels']) if object['custom_labels']
+        sidekiq_metrics.delete_if { |metric| metric['created_at'] + MAX_SIDEKIQ_METRIC_AGE < now }
+        sidekiq_metrics << queue
+      end
+    end
+  end
+end