prometheus_exporter 0.7.0 → 2.3.0

data/lib/prometheus_exporter/instrumentation/puma.rb

@@ -4,39 +4,46 @@ require "json"
 
 # collects stats from puma
 module PrometheusExporter::Instrumentation
-  class Puma
-    def self.start(client: nil, frequency: 30)
-      puma_collector = new
+  class Puma < PeriodicStats
+    def self.start(client: nil, frequency: 30, labels: {})
+      puma_collector = new(labels)
       client ||= PrometheusExporter::Client.default
-      Thread.new do
-        while true
-          begin
-            metric = puma_collector.collect
-            client.send_json metric
-          rescue => e
-            STDERR.puts("Prometheus Exporter Failed To Collect Puma Stats #{e}")
-          ensure
-            sleep frequency
-          end
-        end
+
+      worker_loop do
+        metric = puma_collector.collect
+        client.send_json metric
       end
+
+      super
+    end
+
+    def initialize(metric_labels = {})
+      @metric_labels = metric_labels
     end
 
     def collect
-      metric = {}
-      metric[:type] = "puma"
+      metric = {
+        pid: pid,
+        type: "puma",
+        hostname: ::PrometheusExporter.hostname,
+        metric_labels: @metric_labels,
+      }
       collect_puma_stats(metric)
       metric
     end
 
+    def pid
+      @pid = ::Process.pid
+    end
+
     def collect_puma_stats(metric)
       stats = JSON.parse(::Puma.stats)
 
       if stats.key?("workers")
         metric[:phase] = stats["phase"]
-        metric[:workers_total] = stats["workers"]
-        metric[:booted_workers_total] = stats["booted_workers"]
-        metric[:old_workers_total] = stats["old_workers"]
+        metric[:workers] = stats["workers"]
+        metric[:booted_workers] = stats["booted_workers"]
+        metric[:old_workers] = stats["old_workers"]
 
         stats["worker_status"].each do |worker|
           next if worker["last_status"].empty?
@@ -50,15 +57,17 @@ module PrometheusExporter::Instrumentation
     private
 
     def collect_worker_status(metric, status)
-      metric[:request_backlog_total] ||= 0
-      metric[:running_threads_total] ||= 0
-      metric[:thread_pool_capacity_total] ||= 0
-      metric[:max_threads_total] ||= 0
+      metric[:request_backlog] ||= 0
+      metric[:running_threads] ||= 0
+      metric[:thread_pool_capacity] ||= 0
+      metric[:max_threads] ||= 0
+      metric[:busy_threads] ||= 0
 
-      metric[:request_backlog_total] += status["backlog"]
-      metric[:running_threads_total] += status["running"]
-      metric[:thread_pool_capacity_total] += status["pool_capacity"]
-      metric[:max_threads_total] += status["max_threads"]
+      metric[:request_backlog] += status["backlog"]
+      metric[:running_threads] += status["running"]
+      metric[:thread_pool_capacity] += status["pool_capacity"]
+      metric[:max_threads] += status["max_threads"]
+      metric[:busy_threads] += status["busy_threads"]
     end
   end
 end

data/lib/prometheus_exporter/instrumentation/resque.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+# collects stats from resque
+module PrometheusExporter::Instrumentation
+  class Resque < PeriodicStats
+    def self.start(client: nil, frequency: 30)
+      resque_collector = new
+      client ||= PrometheusExporter::Client.default
+
+      worker_loop { client.send_json(resque_collector.collect) }
+
+      super
+    end
+
+    def collect
+      metric = {}
+      metric[:type] = "resque"
+      collect_resque_stats(metric)
+      metric
+    end
+
+    def collect_resque_stats(metric)
+      info = ::Resque.info
+
+      metric[:processed_jobs] = info[:processed]
+      metric[:failed_jobs] = info[:failed]
+      metric[:pending_jobs] = info[:pending]
+      metric[:queues] = info[:queues]
+      metric[:worker] = info[:workers]
+      metric[:working] = info[:working]
+    end
+  end
+end

data/lib/prometheus_exporter/instrumentation/shoryuken.rb

@@ -2,7 +2,6 @@
 
 module PrometheusExporter::Instrumentation
   class Shoryuken
-
     def initialize(client: nil)
       @client = client || PrometheusExporter::Client.default
     end
@@ -19,12 +18,12 @@ module PrometheusExporter::Instrumentation
     ensure
       duration = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - start
       @client.send_json(
-          type: "shoryuken",
-          queue: queue,
-          name: worker.class.name,
-          success: success,
-          shutdown: shutdown,
-          duration: duration
+        type: "shoryuken",
+        queue: queue,
+        name: worker.class.name,
+        success: success,
+        shutdown: shutdown,
+        duration: duration,
       )
     end
   end

data/lib/prometheus_exporter/instrumentation/sidekiq.rb

@@ -1,32 +1,49 @@
 # frozen_string_literal: true
 
-require 'yaml'
+require "yaml"
 
 module PrometheusExporter::Instrumentation
-  JOB_WRAPPER_CLASS_NAME = 'ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper'
-  DELAYED_CLASS_NAMES = [
-    'Sidekiq::Extensions::DelayedClass',
-    'Sidekiq::Extensions::DelayedModel',
-    'Sidekiq::Extensions::DelayedMailer',
+  JOB_WRAPPER_CLASS_NAME = "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
+  DELAYED_CLASS_NAMES = %w[
+    Sidekiq::Extensions::DelayedClass
+    Sidekiq::Extensions::DelayedModel
+    Sidekiq::Extensions::DelayedMailer
   ]
 
   class Sidekiq
     def self.death_handler
-      -> (job, ex) do
+      ->(job, ex) do
         job_is_fire_and_forget = job["retry"] == false
 
+        worker_class = Object.const_get(job["class"])
+        worker_custom_labels = self.get_worker_custom_labels(worker_class, job)
+
         unless job_is_fire_and_forget
           PrometheusExporter::Client.default.send_json(
             type: "sidekiq",
-            name: job["class"],
+            name: get_name(job["class"], job),
             dead: true,
+            custom_labels: worker_custom_labels,
           )
         end
       end
     end
 
-    def initialize(client: nil)
-      @client = client || PrometheusExporter::Client.default
+    def self.get_worker_custom_labels(worker_class, msg)
+      return {} unless worker_class.respond_to?(:custom_labels)
+
+      # TODO remove when version 3.0.0 is released
+      method_arity = worker_class.method(:custom_labels).arity
+
+      if method_arity > 0
+        worker_class.custom_labels(msg)
+      else
+        worker_class.custom_labels
+      end
+    end
+
+    def initialize(options = { client: nil })
+      @client = options.fetch(:client, nil) || PrometheusExporter::Client.default
     end
 
     def call(worker, msg, queue)
@@ -43,18 +60,18 @@ module PrometheusExporter::Instrumentation
       duration = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - start
       @client.send_json(
         type: "sidekiq",
-        name: get_name(worker, msg),
+        name: self.class.get_name(worker.class.to_s, msg),
         queue: queue,
         success: success,
         shutdown: shutdown,
-        duration: duration
+        duration: duration,
+        custom_labels: self.class.get_worker_custom_labels(worker.class, msg),
       )
     end
 
     private
 
-    def get_name(worker, msg)
-      class_name = worker.class.to_s
+    def self.get_name(class_name, msg)
       if class_name == JOB_WRAPPER_CLASS_NAME
         get_job_wrapper_name(msg)
       elsif DELAYED_CLASS_NAMES.include?(class_name)
@@ -64,24 +81,35 @@ module PrometheusExporter::Instrumentation
       end
     end
 
-    def get_job_wrapper_name(msg)
-      msg['wrapped']
+    def self.get_job_wrapper_name(msg)
+      msg["wrapped"]
     end
 
-    def get_delayed_name(msg, class_name)
-      # fallback to class_name since we're relying on the internal implementation
-      # of the delayed extensions
-      # https://github.com/mperham/sidekiq/blob/master/lib/sidekiq/extensions/class_methods.rb
+    def self.get_delayed_name(msg, class_name)
       begin
-        (target, method_name, _args) = YAML.load(msg['args'].first)
+        # fallback to class_name since we're relying on the internal implementation
+        # of the delayed extensions
+        # https://github.com/mperham/sidekiq/blob/master/lib/sidekiq/extensions/class_methods.rb
+        target, method_name, _args = YAML.load(msg["args"].first)
         if target.class == Class
           "#{target.name}##{method_name}"
         else
           "#{target.class.name}##{method_name}"
         end
-      rescue
-        class_name
+      rescue Psych::DisallowedClass, ArgumentError
+        parsed = Psych.parse(msg["args"].first)
+        children = parsed.root.children
+        target = (children[0].value || children[0].tag).sub("!", "")
+        method_name = (children[1].value || children[1].tag).sub(":", "")
+
+        if target && method_name
+          "#{target}##{method_name}"
+        else
+          class_name
+        end
       end
+    rescue StandardError
+      class_name
     end
   end
 end

data/lib/prometheus_exporter/instrumentation/sidekiq_process.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module PrometheusExporter::Instrumentation
+  class SidekiqProcess < PeriodicStats
+    def self.start(client: nil, frequency: 30)
+      client ||= PrometheusExporter::Client.default
+      sidekiq_process_collector = new
+
+      worker_loop { client.send_json(sidekiq_process_collector.collect) }
+
+      super
+    end
+
+    def initialize
+      @pid = ::Process.pid
+      @hostname = Socket.gethostname
+    end
+
+    def collect
+      { type: "sidekiq_process", process: collect_stats }
+    end
+
+    def collect_stats
+      process = current_process
+      return {} unless process
+
+      {
+        busy: process["busy"],
+        concurrency: process["concurrency"],
+        labels: {
+          labels: process["labels"].sort.join(","),
+          queues: process["queues"].sort.join(","),
+          quiet: process["quiet"],
+          tag: process["tag"],
+          hostname: process["hostname"],
+          identity: process["identity"],
+        },
+      }
+    end
+
+    def current_process
+      ::Sidekiq::ProcessSet.new.find { |sp| sp["hostname"] == @hostname && sp["pid"] == @pid }
+    end
+  end
+end

data/lib/prometheus_exporter/instrumentation/sidekiq_queue.rb

@@ -1,50 +1,55 @@
 # frozen_string_literal: true
 
 module PrometheusExporter::Instrumentation
-  class SidekiqQueue
-    def self.start(client: nil, frequency: 30)
+  class SidekiqQueue < PeriodicStats
+    def self.start(client: nil, frequency: 30, all_queues: false)
       client ||= PrometheusExporter::Client.default
-      sidekiq_queue_collector = new
-
-      Thread.new do
-        loop do
-          begin
-            client.send_json(sidekiq_queue_collector.collect)
-          rescue StandardError => e
-            STDERR.puts("Prometheus Exporter Failed To Collect Sidekiq Queue metrics #{e}")
-          ensure
-            sleep frequency
-          end
-        end
-      end
+      sidekiq_queue_collector = new(all_queues: all_queues)
+
+      worker_loop { client.send_json(sidekiq_queue_collector.collect) }
+
+      super
+    end
+
+    def initialize(all_queues: false)
+      @all_queues = all_queues
+      @pid = ::Process.pid
+      @hostname = Socket.gethostname
     end
 
     def collect
-      {
-        type: 'sidekiq_queue',
-        queues: collect_queue_stats
-      }
+      { type: "sidekiq_queue", queues: collect_queue_stats }
     end
 
     def collect_queue_stats
-      hostname = Socket.gethostname
-      pid = ::Process.pid
-      ps = ::Sidekiq::ProcessSet.new
+      sidekiq_queues = ::Sidekiq::Queue.all
 
-      process = ps.find do |sp|
-        sp['hostname'] == hostname && sp['pid'] == pid
+      unless @all_queues
+        queues = collect_current_process_queues
+        sidekiq_queues.select! { |sidekiq_queue| queues.include?(sidekiq_queue.name) }
       end
 
-      queues = process.nil? ? [] : process['queues']
+      sidekiq_queues
+        .map do |queue|
+          {
+            backlog: queue.size,
+            latency_seconds: queue.latency.to_i,
+            labels: {
+              queue: queue.name,
+            },
+          }
+        end
+        .compact
+    end
+
+    private
+
+    def collect_current_process_queues
+      ps = ::Sidekiq::ProcessSet.new
+
+      process = ps.find { |sp| sp["hostname"] == @hostname && sp["pid"] == @pid }
 
-      ::Sidekiq::Queue.all.map do |queue|
-        next unless queues.include? queue.name
-        {
-          backlog_total: queue.size,
-          latency_seconds: queue.latency.to_i,
-          labels: { queue: queue.name }
-        }
-      end.compact
+      process.nil? ? [] : process["queues"]
     end
   end
 end

data/lib/prometheus_exporter/instrumentation/sidekiq_stats.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module PrometheusExporter::Instrumentation
+  class SidekiqStats < PeriodicStats
+    def self.start(client: nil, frequency: 30)
+      client ||= PrometheusExporter::Client.default
+      sidekiq_stats_collector = new
+
+      worker_loop { client.send_json(sidekiq_stats_collector.collect) }
+
+      super
+    end
+
+    def collect
+      { type: "sidekiq_stats", stats: collect_stats }
+    end
+
+    def collect_stats
+      stats = ::Sidekiq::Stats.new
+      {
+        "dead_size" => stats.dead_size,
+        "enqueued" => stats.enqueued,
+        "failed" => stats.failed,
+        "processed" => stats.processed,
+        "processes_size" => stats.processes_size,
+        "retry_size" => stats.retry_size,
+        "scheduled_size" => stats.scheduled_size,
+        "workers_size" => stats.workers_size,
+      }
+    end
+  end
+end

data/lib/prometheus_exporter/instrumentation/unicorn.rb

@@ -1,29 +1,24 @@
 # frozen_string_literal: true
 
 begin
-  require 'raindrops'
+  require "raindrops"
 rescue LoadError
   # No raindrops available, dont do anything
 end
 
 module PrometheusExporter::Instrumentation
   # collects stats from unicorn
-  class Unicorn
+  class Unicorn < PeriodicStats
     def self.start(pid_file:, listener_address:, client: nil, frequency: 30)
       unicorn_collector = new(pid_file: pid_file, listener_address: listener_address)
       client ||= PrometheusExporter::Client.default
-      Thread.new do
-        loop do
-          begin
-            metric = unicorn_collector.collect
-            client.send_json metric
-          rescue StandardError => e
-            STDERR.puts("Prometheus Exporter Failed To Collect Unicorn Stats #{e}")
-          ensure
-            sleep frequency
-          end
-        end
+
+      worker_loop do
+        metric = unicorn_collector.collect
+        client.send_json metric
       end
+
+      super
     end
 
     def initialize(pid_file:, listener_address:)
@@ -34,7 +29,7 @@ module PrometheusExporter::Instrumentation
 
     def collect
       metric = {}
-      metric[:type] = 'unicorn'
+      metric[:type] = "unicorn"
       collect_unicorn_stats(metric)
       metric
     end
@@ -42,9 +37,9 @@ module PrometheusExporter::Instrumentation
     def collect_unicorn_stats(metric)
       stats = listener_address_stats
 
-      metric[:active_workers_total] = stats.active
-      metric[:request_backlog_total] = stats.queued
-      metric[:workers_total] = worker_process_count
+      metric[:active_workers] = stats.active
+      metric[:request_backlog] = stats.queued
+      metric[:workers] = worker_process_count
     end
 
     private

data/lib/prometheus_exporter/instrumentation.rb

@@ -1,13 +1,18 @@
 # frozen_string_literal: true
 
 require_relative "client"
+require_relative "instrumentation/periodic_stats"
 require_relative "instrumentation/process"
 require_relative "instrumentation/method_profiler"
 require_relative "instrumentation/sidekiq"
 require_relative "instrumentation/sidekiq_queue"
+require_relative "instrumentation/sidekiq_process"
+require_relative "instrumentation/sidekiq_stats"
 require_relative "instrumentation/delayed_job"
 require_relative "instrumentation/puma"
 require_relative "instrumentation/hutch"
 require_relative "instrumentation/unicorn"
 require_relative "instrumentation/active_record"
 require_relative "instrumentation/shoryuken"
+require_relative "instrumentation/resque"
+require_relative "instrumentation/good_job"

data/lib/prometheus_exporter/metric/base.rb

@@ -2,9 +2,9 @@
 
 module PrometheusExporter::Metric
   class Base
-
     @default_prefix = nil if !defined?(@default_prefix)
     @default_labels = nil if !defined?(@default_labels)
+    @default_aggregation = nil if !defined?(@default_aggregation)
 
     # prefix applied to all metrics
     def self.default_prefix=(name)
@@ -23,6 +23,14 @@ module PrometheusExporter::Metric
       @default_labels || {}
     end
 
+    def self.default_aggregation=(aggregation)
+      @default_aggregation = aggregation
+    end
+
+    def self.default_aggregation
+      @default_aggregation ||= Summary
+    end
+
     attr_accessor :help, :name, :data
 
     def initialize(name, help)
@@ -66,13 +74,16 @@ module PrometheusExporter::Metric
     end
 
     def labels_text(labels)
-      labels = (labels || {}).merge(Base.default_labels)
+      labels = Base.default_labels.merge(labels || {})
       if labels && labels.length > 0
-        s = labels.map do |key, value|
-          value = value.to_s
-          value = escape_value(value) if needs_escape?(value)
-          "#{key}=\"#{value}\""
-        end.join(",")
+        s =
+          labels
+            .map do |key, value|
+              value = value.to_s
+              value = escape_value(value) if needs_escape?(value)
+              "#{key}=\"#{value}\""
+            end
+            .join(",")
         "{#{s}}"
       end
     end
@@ -97,16 +108,8 @@ module PrometheusExporter::Metric
       end
     end
 
-    # when we drop Ruby 2.3 we can drop this
-    if "".respond_to? :match?
-      def needs_escape?(str)
-        str.match?(/[\n"\\]/m)
-      end
-    else
-      def needs_escape?(str)
-        !!str.match(/[\n"\\]/m)
-      end
+    def needs_escape?(str)
+      str.match?(/[\n"\\]/m)
     end
-
   end
 end

data/lib/prometheus_exporter/metric/counter.rb

@@ -18,9 +18,7 @@ module PrometheusExporter::Metric
     end
 
     def metric_text
-      @data.map do |labels, value|
-        "#{prefix(@name)}#{labels_text(labels)} #{value}"
-      end.join("\n")
+      @data.map { |labels, value| "#{prefix(@name)}#{labels_text(labels)} #{value}" }.join("\n")
     end
 
     def to_h

data/lib/prometheus_exporter/metric/gauge.rb

@@ -5,6 +5,10 @@ module PrometheusExporter::Metric
     attr_reader :data
 
     def initialize(name, help)
+      if name.end_with?("_total")
+        raise ArgumentError, "The metric name of gauge must not have _total suffix. Given: #{name}"
+      end
+
       super
       reset!
     end
@@ -14,9 +18,7 @@ module PrometheusExporter::Metric
     end
 
     def metric_text
-      @data.map do |labels, value|
-        "#{prefix(@name)}#{labels_text(labels)} #{value}"
-      end.join("\n")
+      @data.map { |labels, value| "#{prefix(@name)}#{labels_text(labels)} #{value}" }.join("\n")
     end
 
     def reset!
@@ -35,9 +37,7 @@ module PrometheusExporter::Metric
       if value.nil?
         data.delete(labels)
       else
-        if !(Numeric === value)
-          raise ArgumentError, 'value must be a number'
-        end
+        raise ArgumentError, "value must be a number" if !(Numeric === value)
         @data[labels] = value
       end
     end