prometheus_exporter 0.5.3 → 0.8.1

data/gemfiles/ar_60.gemfile

@@ -0,0 +1,5 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gemspec path: "../"

data/gemfiles/ar_61.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activerecord", "~> 6.1.0"
+
+gemspec path: "../"

data/lib/prometheus_exporter.rb

@@ -11,6 +11,7 @@ module PrometheusExporter
   DEFAULT_PREFIX = 'ruby_'
   DEFAULT_LABEL = {}
   DEFAULT_TIMEOUT = 2
+  DEFAULT_REALM = 'Prometheus Exporter'
 
   class OjCompat
     def self.parse(obj)

data/lib/prometheus_exporter/client.rb

@@ -2,6 +2,7 @@
 
 require 'socket'
 require 'thread'
+require 'logger'
 
 module PrometheusExporter
   class Client
@@ -53,24 +54,32 @@ module PrometheusExporter
     MAX_SOCKET_AGE = 25
     MAX_QUEUE_SIZE = 10_000
 
+    attr_reader :logger
+
     def initialize(
       host: ENV.fetch('PROMETHEUS_EXPORTER_HOST', 'localhost'),
       port: ENV.fetch('PROMETHEUS_EXPORTER_PORT', PrometheusExporter::DEFAULT_PORT),
       max_queue_size: nil,
       thread_sleep: 0.5,
       json_serializer: nil,
-      custom_labels: nil
+      custom_labels: nil,
+      logger: Logger.new(STDERR),
+      log_level: Logger::WARN
     )
+      @logger = logger
+      @logger.level = log_level
       @metrics = []
 
       @queue = Queue.new
+
       @socket = nil
       @socket_started = nil
+      @socket_pid = nil
 
       max_queue_size ||= MAX_QUEUE_SIZE
       max_queue_size = max_queue_size.to_i
 
-      if max_queue_size.to_i <= 0
+      if max_queue_size <= 0
         raise ArgumentError, "max_queue_size must be larger than 0"
       end
 
@@ -107,14 +116,23 @@ module PrometheusExporter
     end
 
     def send_json(obj)
-      payload = @custom_labels.nil? ? obj : obj.merge(custom_labels: @custom_labels)
+      payload =
+        if @custom_labels
+          if obj[:custom_labels]
+            obj.merge(custom_labels: @custom_labels.merge(obj[:custom_labels]))
+          else
+            obj.merge(custom_labels: @custom_labels)
+          end
+        else
+          obj
+        end
       send(@json_serializer.dump(payload))
     end
 
     def send(str)
       @queue << str
       if @queue.length > @max_queue_size
-        STDERR.puts "Prometheus Exporter client is dropping message cause queue is full"
+        logger.warn "Prometheus Exporter client is dropping message cause queue is full"
         @queue.pop
       end
 
@@ -132,7 +150,7 @@ module PrometheusExporter
           @socket.write(message)
           @socket.write("\r\n")
         rescue => e
-          STDERR.puts "Prometheus Exporter is dropping a message: #{e}"
+          logger.warn "Prometheus Exporter is dropping a message: #{e}"
           @socket = nil
           raise
         end
@@ -157,7 +175,7 @@ module PrometheusExporter
       close_socket_if_old!
       process_queue
     rescue => e
-      STDERR.puts "Prometheus Exporter, failed to send message #{e}"
+      logger.error "Prometheus Exporter, failed to send message #{e}"
     end
 
     def ensure_worker_thread!
@@ -173,11 +191,14 @@ module PrometheusExporter
           end
         end
       end
+    rescue ThreadError => e
+      raise unless e.message =~ /can't alloc thread/
+      logger.error "Prometheus Exporter, failed to send message ThreadError #{e}"
     end
 
     def close_socket!
       begin
-        if @socket
+        if @socket && !@socket.closed?
           @socket.write("0\r\n")
           @socket.write("\r\n")
           @socket.flush
@@ -191,12 +212,20 @@ module PrometheusExporter
     end
 
     def close_socket_if_old!
-      if @socket && ((@socket_started + MAX_SOCKET_AGE) < Time.now.to_f)
+      if @socket_pid == Process.pid && @socket && @socket_started && ((@socket_started + MAX_SOCKET_AGE) < Time.now.to_f)
         close_socket!
       end
     end
 
     def ensure_socket!
+      # if process was forked socket may be owned by parent
+      # leave it alone and reset
+      if @socket_pid != Process.pid
+        @socket = nil
+        @socket_started = nil
+        @socket_pid = nil
+      end
+
       close_socket_if_old!
       if !@socket
         @socket = TCPSocket.new @host, @port
@@ -207,12 +236,14 @@ module PrometheusExporter
         @socket.write("Content-Type: application/octet-stream\r\n")
         @socket.write("\r\n")
         @socket_started = Time.now.to_f
+        @socket_pid = Process.pid
       end
 
       nil
     rescue
       @socket = nil
       @socket_started = nil
+      @socket_pid = nil
       raise
     end
 

data/lib/prometheus_exporter/instrumentation.rb

@@ -11,3 +11,4 @@ require_relative "instrumentation/hutch"
 require_relative "instrumentation/unicorn"
 require_relative "instrumentation/active_record"
 require_relative "instrumentation/shoryuken"
+require_relative "instrumentation/resque"

data/lib/prometheus_exporter/instrumentation/active_record.rb

@@ -7,9 +7,11 @@ module PrometheusExporter::Instrumentation
 
     def self.start(client: nil, frequency: 30, custom_labels: {}, config_labels: [])
 
-      # Not all rails versions support coonection pool stats
+      client ||= PrometheusExporter::Client.default
+
+      # Not all rails versions support connection pool stats
       unless ::ActiveRecord::Base.connection_pool.respond_to?(:stat)
-        STDERR.puts("ActiveRecord connection pool stats not supported in your rails version")
+        client.logger.error("ActiveRecord connection pool stats not supported in your rails version")
         return
       end
 
@@ -18,8 +20,6 @@ module PrometheusExporter::Instrumentation
 
       active_record_collector = new(custom_labels, config_labels)
 
-      client ||= PrometheusExporter::Client.default
-
       stop if @thread
 
       @thread = Thread.new do
@@ -28,7 +28,7 @@ module PrometheusExporter::Instrumentation
             metrics = active_record_collector.collect
             metrics.each { |metric| client.send_json metric }
           rescue => e
-            STDERR.puts("Prometheus Exporter Failed To Collect Process Stats #{e}")
+            client.logger.error("Prometheus Exporter Failed To Collect Process Stats #{e}")
           ensure
             sleep frequency
           end
@@ -67,21 +67,28 @@ module PrometheusExporter::Instrumentation
       ObjectSpace.each_object(::ActiveRecord::ConnectionAdapters::ConnectionPool) do |pool|
         next if pool.connections.nil?
 
-        labels_from_config = pool.spec.config
-          .select { |k, v| @config_labels.include? k }
-          .map { |k, v| [k.to_s.prepend("dbconfig_"), v] }
-
-        labels = @metric_labels.merge(pool_name: pool.spec.name).merge(Hash[labels_from_config])
-
         metric = {
           pid: pid,
           type: "active_record",
           hostname: ::PrometheusExporter.hostname,
-          metric_labels: labels
+          metric_labels: labels(pool)
         }
         metric.merge!(pool.stat)
         metrics << metric
       end
     end
+
+    private
+
+    def labels(pool)
+      if ::ActiveRecord.version < Gem::Version.new("6.1.0.rc1")
+        @metric_labels.merge(pool_name: pool.spec.name).merge(pool.spec.config
+          .select { |k, v| @config_labels.include? k }
+          .map { |k, v| [k.to_s.dup.prepend("dbconfig_"), v] }.to_h)
+      else
+        @metric_labels.merge(pool_name: pool.db_config.name).merge(
+          @config_labels.each_with_object({}) { |l, acc| acc["dbconfig_#{l}"] = pool.db_config.public_send(l) })
+      end
+    end
   end
 end

data/lib/prometheus_exporter/instrumentation/delayed_job.rb

@@ -13,8 +13,8 @@ module PrometheusExporter::Instrumentation
           callbacks do |lifecycle|
             lifecycle.around(:invoke_job) do |job, *args, &block|
               max_attempts = Delayed::Worker.max_attempts
-              enqueued_count = Delayed::Job.count
-              pending_count = Delayed::Job.where(attempts: 0, locked_at: nil).count
+              enqueued_count = Delayed::Job.where(queue: job.queue).count
+              pending_count = Delayed::Job.where(attempts: 0, locked_at: nil, queue: job.queue).count
               instrumenter.call(job, max_attempts, enqueued_count, pending_count, *args, &block)
             end
           end
@@ -41,6 +41,7 @@ module PrometheusExporter::Instrumentation
       @client.send_json(
         type: "delayed_job",
         name: job.handler.to_s.match(JOB_CLASS_REGEXP).to_a[1].to_s,
+        queue_name: job.queue,
         success: success,
         duration: duration,
         attempts: attempts,

data/lib/prometheus_exporter/instrumentation/method_profiler.rb

@@ -5,6 +5,7 @@ module PrometheusExporter::Instrumentation; end
 
 class PrometheusExporter::Instrumentation::MethodProfiler
   def self.patch(klass, methods, name)
+    patch_source_line = __LINE__ + 3
     patches = methods.map do |method_name|
       <<~RUBY
       unless defined?(#{method_name}__mp_unpatched)
@@ -26,7 +27,7 @@ class PrometheusExporter::Instrumentation::MethodProfiler
       RUBY
     end.join("\n")
 
-    klass.class_eval patches
+    klass.class_eval patches, __FILE__, patch_source_line
   end
 
   def self.transfer

data/lib/prometheus_exporter/instrumentation/process.rb

@@ -27,7 +27,7 @@ module PrometheusExporter::Instrumentation
             metric = process_collector.collect
             client.send_json metric
           rescue => e
-            STDERR.puts("Prometheus Exporter Failed To Collect Process Stats #{e}")
+            client.logger.error("Prometheus Exporter Failed To Collect Process Stats #{e}")
           ensure
             sleep frequency
           end

data/lib/prometheus_exporter/instrumentation/puma.rb

@@ -5,8 +5,8 @@ require "json"
 # collects stats from puma
 module PrometheusExporter::Instrumentation
   class Puma
-    def self.start(client: nil, frequency: 30)
-      puma_collector = new
+    def self.start(client: nil, frequency: 30, labels: {})
+      puma_collector = new(labels)
       client ||= PrometheusExporter::Client.default
       Thread.new do
         while true
@@ -14,7 +14,7 @@ module PrometheusExporter::Instrumentation
             metric = puma_collector.collect
             client.send_json metric
           rescue => e
-            STDERR.puts("Prometheus Exporter Failed To Collect Puma Stats #{e}")
+            client.logger.error("Prometheus Exporter Failed To Collect Puma Stats #{e}")
           ensure
             sleep frequency
           end
@@ -22,13 +22,25 @@ module PrometheusExporter::Instrumentation
       end
     end
 
+    def initialize(metric_labels = {})
+      @metric_labels = metric_labels
+    end
+
     def collect
-      metric = {}
-      metric[:type] = "puma"
+      metric = {
+        pid: pid,
+        type: "puma",
+        hostname: ::PrometheusExporter.hostname,
+        metric_labels: @metric_labels
+      }
       collect_puma_stats(metric)
       metric
     end
 
+    def pid
+      @pid = ::Process.pid
+    end
+
     def collect_puma_stats(metric)
       stats = JSON.parse(::Puma.stats)
 

data/lib/prometheus_exporter/instrumentation/resque.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+# collects stats from resque
+module PrometheusExporter::Instrumentation
+  class Resque
+    def self.start(client: nil, frequency: 30)
+      resque_collector = new
+      client ||= PrometheusExporter::Client.default
+      Thread.new do
+        while true
+          begin
+            client.send_json(resque_collector.collect)
+          rescue => e
+            client.logger.error("Prometheus Exporter Failed To Collect Resque Stats #{e}")
+          ensure
+            sleep frequency
+          end
+        end
+      end
+    end
+
+    def collect
+      metric = {}
+      metric[:type] = "resque"
+      collect_resque_stats(metric)
+      metric
+    end
+
+    def collect_resque_stats(metric)
+      info = ::Resque.info
+
+      metric[:processed_jobs_total] = info[:processed]
+      metric[:failed_jobs_total] = info[:failed]
+      metric[:pending_jobs_total] = info[:pending]
+      metric[:queues_total] = info[:queues]
+      metric[:worker_total] = info[:workers]
+      metric[:working_total] = info[:working]
+    end
+  end
+end

data/lib/prometheus_exporter/instrumentation/sidekiq.rb

@@ -1,6 +1,15 @@
 # frozen_string_literal: true
 
+require 'yaml'
+
 module PrometheusExporter::Instrumentation
+  JOB_WRAPPER_CLASS_NAME = 'ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper'
+  DELAYED_CLASS_NAMES = [
+    'Sidekiq::Extensions::DelayedClass',
+    'Sidekiq::Extensions::DelayedModel',
+    'Sidekiq::Extensions::DelayedMailer',
+  ]
+
   class Sidekiq
     def self.death_handler
       -> (job, ex) do
@@ -32,15 +41,47 @@ module PrometheusExporter::Instrumentation
       raise e
     ensure
       duration = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - start
-      class_name = worker.class.to_s == 'ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper' ?
-                     msg['wrapped'] : worker.class.to_s
       @client.send_json(
         type: "sidekiq",
-        name: class_name,
+        name: get_name(worker, msg),
+        queue: queue,
         success: success,
         shutdown: shutdown,
         duration: duration
       )
     end
+
+    private
+
+    def get_name(worker, msg)
+      class_name = worker.class.to_s
+      if class_name == JOB_WRAPPER_CLASS_NAME
+        get_job_wrapper_name(msg)
+      elsif DELAYED_CLASS_NAMES.include?(class_name)
+        get_delayed_name(msg, class_name)
+      else
+        class_name
+      end
+    end
+
+    def get_job_wrapper_name(msg)
+      msg['wrapped']
+    end
+
+    def get_delayed_name(msg, class_name)
+      # fallback to class_name since we're relying on the internal implementation
+      # of the delayed extensions
+      # https://github.com/mperham/sidekiq/blob/master/lib/sidekiq/extensions/class_methods.rb
+      begin
+        (target, method_name, _args) = YAML.load(msg['args'].first)
+        if target.class == Class
+          "#{target.name}##{method_name}"
+        else
+          "#{target.class.name}##{method_name}"
+        end
+      rescue
+        class_name
+      end
+    end
   end
 end

data/lib/prometheus_exporter/instrumentation/sidekiq_queue.rb

@@ -11,7 +11,7 @@ module PrometheusExporter::Instrumentation
           begin
             client.send_json(sidekiq_queue_collector.collect)
           rescue StandardError => e
-            STDERR.puts("Prometheus Exporter Failed To Collect Sidekiq Queue metrics #{e}")
+            client.logger.error("Prometheus Exporter Failed To Collect Sidekiq Queue metrics #{e}")
           ensure
             sleep frequency
           end
@@ -27,13 +27,24 @@ module PrometheusExporter::Instrumentation
     end
 
     def collect_queue_stats
+      hostname = Socket.gethostname
+      pid = ::Process.pid
+      ps = ::Sidekiq::ProcessSet.new
+
+      process = ps.find do |sp|
+        sp['hostname'] == hostname && sp['pid'] == pid
+      end
+
+      queues = process.nil? ? [] : process['queues']
+
       ::Sidekiq::Queue.all.map do |queue|
+        next unless queues.include? queue.name
         {
           backlog_total: queue.size,
           latency_seconds: queue.latency.to_i,
           labels: { queue: queue.name }
         }
-      end
+      end.compact
     end
   end
 end