prometheus_exporter 0.8.0 → 2.0.0

data/lib/prometheus_exporter/client.rb

@@ -2,6 +2,7 @@
 
 require 'socket'
 require 'thread'
+require 'logger'
 
 module PrometheusExporter
   class Client
@@ -53,14 +54,20 @@ module PrometheusExporter
     MAX_SOCKET_AGE = 25
     MAX_QUEUE_SIZE = 10_000
 
+    attr_reader :logger
+
     def initialize(
       host: ENV.fetch('PROMETHEUS_EXPORTER_HOST', 'localhost'),
       port: ENV.fetch('PROMETHEUS_EXPORTER_PORT', PrometheusExporter::DEFAULT_PORT),
       max_queue_size: nil,
       thread_sleep: 0.5,
       json_serializer: nil,
-      custom_labels: nil
+      custom_labels: nil,
+      logger: Logger.new(STDERR),
+      log_level: Logger::WARN
     )
+      @logger = logger
+      @logger.level = log_level
       @metrics = []
 
       @queue = Queue.new
@@ -72,7 +79,7 @@ module PrometheusExporter
       max_queue_size ||= MAX_QUEUE_SIZE
       max_queue_size = max_queue_size.to_i
 
-      if max_queue_size.to_i <= 0
+      if max_queue_size <= 0
         raise ArgumentError, "max_queue_size must be larger than 0"
       end
 
@@ -125,7 +132,7 @@ module PrometheusExporter
     def send(str)
       @queue << str
       if @queue.length > @max_queue_size
-        STDERR.puts "Prometheus Exporter client is dropping message cause queue is full"
+        logger.warn "Prometheus Exporter client is dropping message cause queue is full"
         @queue.pop
       end
 
@@ -143,7 +150,7 @@ module PrometheusExporter
           @socket.write(message)
           @socket.write("\r\n")
         rescue => e
-          STDERR.puts "Prometheus Exporter is dropping a message: #{e}"
+          logger.warn "Prometheus Exporter is dropping a message: #{e}"
           @socket = nil
           raise
         end
@@ -168,7 +175,7 @@ module PrometheusExporter
       close_socket_if_old!
       process_queue
     rescue => e
-      STDERR.puts "Prometheus Exporter, failed to send message #{e}"
+      logger.error "Prometheus Exporter, failed to send message #{e}"
     end
 
     def ensure_worker_thread!
@@ -186,7 +193,7 @@ module PrometheusExporter
       end
     rescue ThreadError => e
       raise unless e.message =~ /can't alloc thread/
-      STDERR.puts "Prometheus Exporter, failed to send message ThreadError #{e}"
+      logger.error "Prometheus Exporter, failed to send message ThreadError #{e}"
     end
 
     def close_socket!

data/lib/prometheus_exporter/instrumentation/active_record.rb

@@ -7,9 +7,11 @@ module PrometheusExporter::Instrumentation
 
     def self.start(client: nil, frequency: 30, custom_labels: {}, config_labels: [])
 
-      # Not all rails versions support coonection pool stats
+      client ||= PrometheusExporter::Client.default
+
+      # Not all rails versions support connection pool stats
       unless ::ActiveRecord::Base.connection_pool.respond_to?(:stat)
-        STDERR.puts("ActiveRecord connection pool stats not supported in your rails version")
+        client.logger.error("ActiveRecord connection pool stats not supported in your rails version")
         return
       end
 
@@ -18,8 +20,6 @@ module PrometheusExporter::Instrumentation
 
       active_record_collector = new(custom_labels, config_labels)
 
-      client ||= PrometheusExporter::Client.default
-
       stop if @thread
 
       @thread = Thread.new do
@@ -28,7 +28,7 @@ module PrometheusExporter::Instrumentation
             metrics = active_record_collector.collect
             metrics.each { |metric| client.send_json metric }
           rescue => e
-            STDERR.puts("Prometheus Exporter Failed To Collect Process Stats #{e}")
+            client.logger.error("Prometheus Exporter Failed To Collect Process Stats #{e}")
           ensure
             sleep frequency
           end

data/lib/prometheus_exporter/instrumentation/process.rb

@@ -27,7 +27,7 @@ module PrometheusExporter::Instrumentation
             metric = process_collector.collect
             client.send_json metric
           rescue => e
-            STDERR.puts("Prometheus Exporter Failed To Collect Process Stats #{e}")
+            client.logger.error("Prometheus Exporter Failed To Collect Process Stats #{e}")
           ensure
             sleep frequency
           end

data/lib/prometheus_exporter/instrumentation/puma.rb

@@ -14,7 +14,7 @@ module PrometheusExporter::Instrumentation
             metric = puma_collector.collect
             client.send_json metric
           rescue => e
-            STDERR.puts("Prometheus Exporter Failed To Collect Puma Stats #{e}")
+            client.logger.error("Prometheus Exporter Failed To Collect Puma Stats #{e}")
           ensure
             sleep frequency
           end
@@ -46,9 +46,9 @@ module PrometheusExporter::Instrumentation
 
       if stats.key?("workers")
         metric[:phase] = stats["phase"]
-        metric[:workers_total] = stats["workers"]
-        metric[:booted_workers_total] = stats["booted_workers"]
-        metric[:old_workers_total] = stats["old_workers"]
+        metric[:workers] = stats["workers"]
+        metric[:booted_workers] = stats["booted_workers"]
+        metric[:old_workers] = stats["old_workers"]
 
         stats["worker_status"].each do |worker|
           next if worker["last_status"].empty?
@@ -62,15 +62,15 @@ module PrometheusExporter::Instrumentation
     private
 
     def collect_worker_status(metric, status)
-      metric[:request_backlog_total] ||= 0
-      metric[:running_threads_total] ||= 0
-      metric[:thread_pool_capacity_total] ||= 0
-      metric[:max_threads_total] ||= 0
+      metric[:request_backlog] ||= 0
+      metric[:running_threads] ||= 0
+      metric[:thread_pool_capacity] ||= 0
+      metric[:max_threads] ||= 0
 
-      metric[:request_backlog_total] += status["backlog"]
-      metric[:running_threads_total] += status["running"]
-      metric[:thread_pool_capacity_total] += status["pool_capacity"]
-      metric[:max_threads_total] += status["max_threads"]
+      metric[:request_backlog] += status["backlog"]
+      metric[:running_threads] += status["running"]
+      metric[:thread_pool_capacity] += status["pool_capacity"]
+      metric[:max_threads] += status["max_threads"]
     end
   end
 end

data/lib/prometheus_exporter/instrumentation/resque.rb

@@ -11,7 +11,7 @@ module PrometheusExporter::Instrumentation
           begin
             client.send_json(resque_collector.collect)
           rescue => e
-            STDERR.puts("Prometheus Exporter Failed To Collect Resque Stats #{e}")
+            client.logger.error("Prometheus Exporter Failed To Collect Resque Stats #{e}")
           ensure
             sleep frequency
           end
@@ -29,12 +29,12 @@ module PrometheusExporter::Instrumentation
     def collect_resque_stats(metric)
       info = ::Resque.info
 
-      metric[:processed_jobs_total] = info[:processed]
-      metric[:failed_jobs_total] = info[:failed]
-      metric[:pending_jobs_total] = info[:pending]
-      metric[:queues_total] = info[:queues]
-      metric[:worker_total] = info[:workers]
-      metric[:working_total] = info[:working]
+      metric[:processed_jobs] = info[:processed]
+      metric[:failed_jobs] = info[:failed]
+      metric[:pending_jobs] = info[:pending]
+      metric[:queues] = info[:queues]
+      metric[:worker] = info[:workers]
+      metric[:working] = info[:working]
     end
   end
 end

data/lib/prometheus_exporter/instrumentation/sidekiq.rb

@@ -15,16 +15,24 @@ module PrometheusExporter::Instrumentation
       -> (job, ex) do
         job_is_fire_and_forget = job["retry"] == false
 
+        worker_class = Object.const_get(job["class"])
+        worker_custom_labels = self.get_worker_custom_labels(worker_class)
+
         unless job_is_fire_and_forget
           PrometheusExporter::Client.default.send_json(
             type: "sidekiq",
             name: job["class"],
             dead: true,
+            custom_labels: worker_custom_labels
           )
         end
       end
     end
 
+    def self.get_worker_custom_labels(worker_class)
+      worker_class.respond_to?(:custom_labels) ? worker_class.custom_labels : {}
+    end
+
     def initialize(client: nil)
       @client = client || PrometheusExporter::Client.default
     end
@@ -47,7 +55,8 @@ module PrometheusExporter::Instrumentation
         queue: queue,
         success: success,
         shutdown: shutdown,
-        duration: duration
+        duration: duration,
+        custom_labels: self.class.get_worker_custom_labels(worker.class)
       )
     end
 
@@ -69,19 +78,30 @@ module PrometheusExporter::Instrumentation
     end
 
     def get_delayed_name(msg, class_name)
-      # fallback to class_name since we're relying on the internal implementation
-      # of the delayed extensions
-      # https://github.com/mperham/sidekiq/blob/master/lib/sidekiq/extensions/class_methods.rb
       begin
-        (target, method_name, _args) = YAML.load(msg['args'].first)
+        # fallback to class_name since we're relying on the internal implementation
+        # of the delayed extensions
+        # https://github.com/mperham/sidekiq/blob/master/lib/sidekiq/extensions/class_methods.rb
+        (target, method_name, _args) = YAML.load(msg['args'].first) # rubocop:disable Security/YAMLLoad
         if target.class == Class
           "#{target.name}##{method_name}"
         else
           "#{target.class.name}##{method_name}"
         end
-      rescue
-        class_name
+      rescue Psych::DisallowedClass, ArgumentError
+        parsed = Psych.parse(msg['args'].first)
+        children = parsed.root.children
+        target = (children[0].value || children[0].tag).sub('!', '')
+        method_name = (children[1].value || children[1].tag).sub(':', '')
+
+        if target && method_name
+          "#{target}##{method_name}"
+        else
+          class_name
+        end
       end
+    rescue
+      class_name
     end
   end
 end

data/lib/prometheus_exporter/instrumentation/sidekiq_process.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module PrometheusExporter::Instrumentation
+  class SidekiqProcess
+    def self.start(client: nil, frequency: 30)
+      client ||= PrometheusExporter::Client.default
+      sidekiq_process_collector = new
+
+      Thread.new do
+        loop do
+          begin
+            client.send_json(sidekiq_process_collector.collect)
+          rescue StandardError => e
+            STDERR.puts("Prometheus Exporter Failed To Collect Sidekiq Processes metrics #{e}")
+          ensure
+            sleep frequency
+          end
+        end
+      end
+    end
+
+    def initialize
+      @pid = ::Process.pid
+      @hostname = Socket.gethostname
+    end
+
+    def collect
+      {
+        type: 'sidekiq_process',
+        process: collect_stats
+      }
+    end
+
+    def collect_stats
+      process = current_process
+      return {} unless process
+
+      {
+        busy: process['busy'],
+        concurrency: process['concurrency'],
+        labels: {
+          labels: process['labels'].sort.join(','),
+          queues: process['queues'].sort.join(','),
+          quiet: process['quiet'],
+          tag: process['tag'],
+          hostname: process['hostname'],
+          identity: process['identity'],
+        }
+      }
+    end
+
+    def current_process
+      ::Sidekiq::ProcessSet.new.find do |sp|
+        sp['hostname'] == @hostname && sp['pid'] == @pid
+      end
+    end
+  end
+end

data/lib/prometheus_exporter/instrumentation/sidekiq_queue.rb

@@ -2,16 +2,16 @@
 
 module PrometheusExporter::Instrumentation
   class SidekiqQueue
-    def self.start(client: nil, frequency: 30)
+    def self.start(client: nil, frequency: 30, all_queues: false)
       client ||= PrometheusExporter::Client.default
-      sidekiq_queue_collector = new
+      sidekiq_queue_collector = new(all_queues: all_queues)
 
       Thread.new do
         loop do
           begin
             client.send_json(sidekiq_queue_collector.collect)
           rescue StandardError => e
-            STDERR.puts("Prometheus Exporter Failed To Collect Sidekiq Queue metrics #{e}")
+            client.logger.error("Prometheus Exporter Failed To Collect Sidekiq Queue metrics #{e}")
           ensure
             sleep frequency
           end
@@ -19,6 +19,12 @@ module PrometheusExporter::Instrumentation
       end
     end
 
+    def initialize(all_queues: false)
+      @all_queues = all_queues
+      @pid = ::Process.pid
+      @hostname = Socket.gethostname
+    end
+
     def collect
       {
         type: 'sidekiq_queue',
@@ -27,24 +33,32 @@ module PrometheusExporter::Instrumentation
     end
 
     def collect_queue_stats
-      hostname = Socket.gethostname
-      pid = ::Process.pid
-      ps = ::Sidekiq::ProcessSet.new
+      sidekiq_queues = ::Sidekiq::Queue.all
 
-      process = ps.find do |sp|
-        sp['hostname'] == hostname && sp['pid'] == pid
+      unless @all_queues
+        queues = collect_current_process_queues
+        sidekiq_queues.select! { |sidekiq_queue| queues.include?(sidekiq_queue.name) }
       end
 
-      queues = process.nil? ? [] : process['queues']
-
-      ::Sidekiq::Queue.all.map do |queue|
-        next unless queues.include? queue.name
+      sidekiq_queues.map do |queue|
         {
-          backlog_total: queue.size,
+          backlog: queue.size,
           latency_seconds: queue.latency.to_i,
           labels: { queue: queue.name }
         }
       end.compact
     end
+
+    private
+
+    def collect_current_process_queues
+      ps = ::Sidekiq::ProcessSet.new
+
+      process = ps.find do |sp|
+        sp['hostname'] == @hostname && sp['pid'] == @pid
+      end
+
+      process.nil? ? [] : process['queues']
+    end
   end
 end

data/lib/prometheus_exporter/instrumentation/sidekiq_stats.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module PrometheusExporter::Instrumentation
+  class SidekiqStats
+    def self.start(client: nil, frequency: 30)
+      client ||= PrometheusExporter::Client.default
+      sidekiq_stats_collector = new
+
+      Thread.new do
+        loop do
+          begin
+            client.send_json(sidekiq_stats_collector.collect)
+          rescue StandardError => e
+            STDERR.puts("Prometheus Exporter Failed To Collect Sidekiq Stats metrics #{e}")
+          ensure
+            sleep frequency
+          end
+        end
+      end
+    end
+
+    def collect
+      {
+        type: 'sidekiq_stats',
+        stats: collect_stats
+      }
+    end
+
+    def collect_stats
+      stats = ::Sidekiq::Stats.new
+      {
+        'dead_size' => stats.dead_size,
+        'enqueued' => stats.enqueued,
+        'failed' => stats.failed,
+        'processed' => stats.processed,
+        'processes_size' => stats.processes_size,
+        'retry_size' => stats.retry_size,
+        'scheduled_size' => stats.scheduled_size,
+        'workers_size' => stats.workers_size,
+      }
+    end
+  end
+end

data/lib/prometheus_exporter/instrumentation/unicorn.rb

@@ -18,7 +18,7 @@ module PrometheusExporter::Instrumentation
             metric = unicorn_collector.collect
             client.send_json metric
           rescue StandardError => e
-            STDERR.puts("Prometheus Exporter Failed To Collect Unicorn Stats #{e}")
+            client.logger.error("Prometheus Exporter Failed To Collect Unicorn Stats #{e}")
           ensure
             sleep frequency
           end
@@ -42,9 +42,9 @@ module PrometheusExporter::Instrumentation
     def collect_unicorn_stats(metric)
       stats = listener_address_stats
 
-      metric[:active_workers_total] = stats.active
-      metric[:request_backlog_total] = stats.queued
-      metric[:workers_total] = worker_process_count
+      metric[:active_workers] = stats.active
+      metric[:request_backlog] = stats.queued
+      metric[:workers] = worker_process_count
     end
 
     private

data/lib/prometheus_exporter/instrumentation.rb

@@ -5,6 +5,8 @@ require_relative "instrumentation/process"
 require_relative "instrumentation/method_profiler"
 require_relative "instrumentation/sidekiq"
 require_relative "instrumentation/sidekiq_queue"
+require_relative "instrumentation/sidekiq_process"
+require_relative "instrumentation/sidekiq_stats"
 require_relative "instrumentation/delayed_job"
 require_relative "instrumentation/puma"
 require_relative "instrumentation/hutch"

data/lib/prometheus_exporter/metric/base.rb

@@ -5,6 +5,7 @@ module PrometheusExporter::Metric
 
     @default_prefix = nil if !defined?(@default_prefix)
     @default_labels = nil if !defined?(@default_labels)
+    @default_aggregation = nil if !defined?(@default_aggregation)
 
     # prefix applied to all metrics
     def self.default_prefix=(name)
@@ -23,6 +24,14 @@ module PrometheusExporter::Metric
       @default_labels || {}
     end
 
+    def self.default_aggregation=(aggregation)
+      @default_aggregation = aggregation
+    end
+
+    def self.default_aggregation
+      @default_aggregation ||= Summary
+    end
+
     attr_accessor :help, :name, :data
 
     def initialize(name, help)
@@ -97,15 +106,8 @@ module PrometheusExporter::Metric
       end
     end
 
-    # when we drop Ruby 2.3 we can drop this
-    if "".respond_to? :match?
-      def needs_escape?(str)
-        str.match?(/[\n"\\]/m)
-      end
-    else
-      def needs_escape?(str)
-        !!str.match(/[\n"\\]/m)
-      end
+    def needs_escape?(str)
+      str.match?(/[\n"\\]/m)
     end
 
   end

data/lib/prometheus_exporter/metric/gauge.rb

@@ -5,6 +5,10 @@ module PrometheusExporter::Metric
     attr_reader :data
 
     def initialize(name, help)
+      if name.end_with?("_total")
+        raise ArgumentError, "The metric name of gauge must not have _total suffix. Given: #{name}"
+      end
+
       super
       reset!
     end

data/lib/prometheus_exporter/metric/histogram.rb

@@ -5,9 +5,21 @@ module PrometheusExporter::Metric
 
     DEFAULT_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5.0, 10.0].freeze
 
+    @default_buckets = nil if !defined?(@default_buckets)
+
+    def self.default_buckets
+      @default_buckets || DEFAULT_BUCKETS
+    end
+
+    def self.default_buckets=(buckets)
+      @default_buckets = buckets
+    end
+
+    attr_reader :buckets
+
     def initialize(name, help, opts = {})
       super(name, help)
-      @buckets = (opts[:buckets] || DEFAULT_BUCKETS).sort.reverse
+      @buckets = (opts[:buckets] || self.class.default_buckets).sort.reverse
       reset!
     end
 

data/lib/prometheus_exporter/middleware.rb

@@ -36,11 +36,12 @@ class PrometheusExporter::Middleware
 
     result
   ensure
-
+    status = (result && result[0]) || -1
     obj = {
       type: "web",
       timings: info,
       queue_time: queue_time,
+      status: status,
       default_labels: default_labels(env, result)
     }
     labels = custom_labels(env)
@@ -52,18 +53,21 @@ class PrometheusExporter::Middleware
   end
 
   def default_labels(env, result)
-    status = (result && result[0]) || -1
     params = env["action_dispatch.request.parameters"]
     action = controller = nil
     if params
       action = params["action"]
       controller = params["controller"]
+    elsif (cors = env["rack.cors"]) && cors.respond_to?(:preflight?) && cors.preflight?
+      # if the Rack CORS Middleware identifies the request as a preflight request,
+      # the stack doesn't get to the point where controllers/actions are defined
+      action = "preflight"
+      controller = "preflight"
     end
 
     {
       action: action || "other",
-      controller: controller || "other",
-      status: status
+      controller: controller || "other"
     }
   end
 

data/lib/prometheus_exporter/server/collector.rb

@@ -14,6 +14,8 @@ module PrometheusExporter::Server
       register_collector(ProcessCollector.new)
       register_collector(SidekiqCollector.new)
       register_collector(SidekiqQueueCollector.new)
+      register_collector(SidekiqProcessCollector.new)
+      register_collector(SidekiqStatsCollector.new)
       register_collector(DelayedJobCollector.new)
       register_collector(PumaCollector.new)
       register_collector(HutchCollector.new)

data/lib/prometheus_exporter/server/delayed_job_collector.rb

@@ -19,22 +19,21 @@ module PrometheusExporter::Server
     end
 
     def collect(obj)
-      default_labels = { job_name: obj['name'], queue_name: obj['queue_name'] }
-      custom_labels = obj['custom_labels']
-
-      labels = custom_labels.nil? ? default_labels : default_labels.merge(custom_labels)
+      custom_labels = obj['custom_labels'] || {}
+      gauge_labels = { queue_name: obj['queue_name'] }.merge(custom_labels)
+      counter_labels = gauge_labels.merge(job_name: obj['name'])
 
       ensure_delayed_job_metrics
-      @delayed_job_duration_seconds.observe(obj["duration"], labels)
-      @delayed_jobs_total.observe(1, labels)
-      @delayed_failed_jobs_total.observe(1, labels) if !obj["success"]
-      @delayed_jobs_max_attempts_reached_total.observe(1, labels) if obj["attempts"] >= obj["max_attempts"]
-      @delayed_job_duration_seconds_summary.observe(obj["duration"], labels)
-      @delayed_job_duration_seconds_summary.observe(obj["duration"], labels.merge(status: "success")) if obj["success"]
-      @delayed_job_duration_seconds_summary.observe(obj["duration"], labels.merge(status: "failed"))  if !obj["success"]
-      @delayed_job_attempts_summary.observe(obj["attempts"], labels) if obj["success"]
-      @delayed_jobs_enqueued.observe(obj["enqueued"], labels)
-      @delayed_jobs_pending.observe(obj["pending"], labels)
+      @delayed_job_duration_seconds.observe(obj["duration"], counter_labels)
+      @delayed_jobs_total.observe(1, counter_labels)
+      @delayed_failed_jobs_total.observe(1, counter_labels) if !obj["success"]
+      @delayed_jobs_max_attempts_reached_total.observe(1, counter_labels) if obj["attempts"] >= obj["max_attempts"]
+      @delayed_job_duration_seconds_summary.observe(obj["duration"], counter_labels)
+      @delayed_job_duration_seconds_summary.observe(obj["duration"], counter_labels.merge(status: "success")) if obj["success"]
+      @delayed_job_duration_seconds_summary.observe(obj["duration"], counter_labels.merge(status: "failed"))  if !obj["success"]
+      @delayed_job_attempts_summary.observe(obj["attempts"], counter_labels) if obj["success"]
+      @delayed_jobs_enqueued.observe(obj["enqueued"], gauge_labels)
+      @delayed_jobs_pending.observe(obj["pending"], gauge_labels)
     end
 
     def metrics
@@ -77,12 +76,12 @@ module PrometheusExporter::Server
                 "delayed_jobs_max_attempts_reached_total", "Total number of delayed jobs that reached max attempts.")
 
         @delayed_job_duration_seconds_summary =
-            PrometheusExporter::Metric::Summary.new("delayed_job_duration_seconds_summary",
-                                                    "Summary of the time it takes jobs to execute.")
+            PrometheusExporter::Metric::Base.default_aggregation.new("delayed_job_duration_seconds_summary",
+                                                                     "Summary of the time it takes jobs to execute.")
 
         @delayed_job_attempts_summary =
-            PrometheusExporter::Metric::Summary.new("delayed_job_attempts_summary",
-                                                    "Summary of the amount of attempts it takes delayed jobs to succeed.")
+            PrometheusExporter::Metric::Base.default_aggregation.new("delayed_job_attempts_summary",
+                                                                     "Summary of the amount of attempts it takes delayed jobs to succeed.")
       end
     end
   end