prometheus_exporter 0.7.0 → 2.1.0

data/lib/prometheus_exporter/client.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require 'socket'
-require 'thread'
+require "socket"
+require "logger"
 
 module PrometheusExporter
   class Client
@@ -24,7 +24,9 @@ module PrometheusExporter
           keys: keys,
           value: value
         }
-        values[:prometheus_exporter_action] = prometheus_exporter_action if prometheus_exporter_action
+        values[
+          :prometheus_exporter_action
+        ] = prometheus_exporter_action if prometheus_exporter_action
         values[:opts] = @opts if @opts
         values
       end
@@ -53,14 +55,23 @@ module PrometheusExporter
     MAX_SOCKET_AGE = 25
     MAX_QUEUE_SIZE = 10_000
 
+    attr_reader :logger
+
     def initialize(
-      host: ENV.fetch('PROMETHEUS_EXPORTER_HOST', 'localhost'),
-      port: ENV.fetch('PROMETHEUS_EXPORTER_PORT', PrometheusExporter::DEFAULT_PORT),
+      host: ENV.fetch("PROMETHEUS_EXPORTER_HOST", "localhost"),
+      port: ENV.fetch(
+        "PROMETHEUS_EXPORTER_PORT",
+        PrometheusExporter::DEFAULT_PORT
+      ),
       max_queue_size: nil,
       thread_sleep: 0.5,
       json_serializer: nil,
-      custom_labels: nil
+      custom_labels: nil,
+      logger: Logger.new(STDERR),
+      log_level: Logger::WARN
     )
+      @logger = logger
+      @logger.level = log_level
       @metrics = []
 
       @queue = Queue.new
@@ -72,7 +83,7 @@ module PrometheusExporter
       max_queue_size ||= MAX_QUEUE_SIZE
       max_queue_size = max_queue_size.to_i
 
-      if max_queue_size.to_i <= 0
+      if max_queue_size <= 0
         raise ArgumentError, "max_queue_size must be larger than 0"
       end
 
@@ -83,7 +94,8 @@ module PrometheusExporter
       @mutex = Mutex.new
       @thread_sleep = thread_sleep
 
-      @json_serializer = json_serializer == :oj ? PrometheusExporter::OjCompat : JSON
+      @json_serializer =
+        json_serializer == :oj ? PrometheusExporter::OjCompat : JSON
 
       @custom_labels = custom_labels
     end
@@ -93,7 +105,14 @@ module PrometheusExporter
     end
 
     def register(type, name, help, opts = nil)
-      metric = RemoteMetric.new(type: type, name: name, help: help, client: self, opts: opts)
+      metric =
+        RemoteMetric.new(
+          type: type,
+          name: name,
+          help: help,
+          client: self,
+          opts: opts
+        )
       @metrics << metric
       metric
     end
@@ -125,7 +144,7 @@ module PrometheusExporter
     def send(str)
       @queue << str
       if @queue.length > @max_queue_size
-        STDERR.puts "Prometheus Exporter client is dropping message cause queue is full"
+        logger.warn "Prometheus Exporter client is dropping message cause queue is full"
         @queue.pop
       end
 
@@ -143,7 +162,7 @@ module PrometheusExporter
           @socket.write(message)
           @socket.write("\r\n")
         rescue => e
-          STDERR.puts "Prometheus Exporter is dropping a message: #{e}"
+          logger.warn "Prometheus Exporter is dropping a message: #{e}"
           @socket = nil
           raise
         end
@@ -154,9 +173,7 @@ module PrometheusExporter
       @mutex.synchronize do
         wait_for_empty_queue_with_timeout(wait_timeout_seconds)
         @worker_thread&.kill
-        while @worker_thread&.alive?
-          sleep 0.001
-        end
+        sleep 0.001 while @worker_thread&.alive?
         @worker_thread = nil
         close_socket!
       end
@@ -168,7 +185,7 @@ module PrometheusExporter
       close_socket_if_old!
       process_queue
     rescue => e
-      STDERR.puts "Prometheus Exporter, failed to send message #{e}"
+      logger.error "Prometheus Exporter, failed to send message #{e}"
     end
 
     def ensure_worker_thread!
@@ -176,14 +193,18 @@ module PrometheusExporter
         @mutex.synchronize do
           return if @worker_thread&.alive?
 
-          @worker_thread = Thread.new do
-            while true
-              worker_loop
-              sleep @thread_sleep
+          @worker_thread =
+            Thread.new do
+              while true
+                worker_loop
+                sleep @thread_sleep
+              end
             end
-          end
         end
       end
+    rescue ThreadError => e
+      raise unless e.message =~ /can't alloc thread/
+      logger.error "Prometheus Exporter, failed to send message ThreadError #{e}"
     end
 
     def close_socket!
@@ -202,7 +223,8 @@ module PrometheusExporter
     end
 
     def close_socket_if_old!
-      if @socket_pid == Process.pid && @socket && @socket_started && ((@socket_started + MAX_SOCKET_AGE) < Time.now.to_f)
+      if @socket_pid == Process.pid && @socket && @socket_started &&
+           ((@socket_started + MAX_SOCKET_AGE) < Time.now.to_f)
         close_socket!
       end
     end
@@ -230,7 +252,7 @@ module PrometheusExporter
       end
 
       nil
-    rescue
+    rescue StandardError
       @socket = nil
       @socket_started = nil
       @socket_pid = nil
@@ -240,7 +262,10 @@ module PrometheusExporter
     def wait_for_empty_queue_with_timeout(timeout_seconds)
       start_time = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
       while @queue.length > 0
-        break if start_time + timeout_seconds < ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+        if start_time + timeout_seconds <
+             ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+          break
+        end
         sleep(0.05)
       end
     end

data/lib/prometheus_exporter/instrumentation/active_record.rb

@@ -2,14 +2,15 @@
 
 # collects stats from currently running process
 module PrometheusExporter::Instrumentation
-  class ActiveRecord
+  class ActiveRecord < PeriodicStats
     ALLOWED_CONFIG_LABELS = %i(database username host port)
 
     def self.start(client: nil, frequency: 30, custom_labels: {}, config_labels: [])
+      client ||= PrometheusExporter::Client.default
 
-      # Not all rails versions support coonection pool stats
+      # Not all rails versions support connection pool stats
       unless ::ActiveRecord::Base.connection_pool.respond_to?(:stat)
-        STDERR.puts("ActiveRecord connection pool stats not supported in your rails version")
+        client.logger.error("ActiveRecord connection pool stats not supported in your rails version")
         return
       end
 
@@ -18,22 +19,12 @@ module PrometheusExporter::Instrumentation
 
       active_record_collector = new(custom_labels, config_labels)
 
-      client ||= PrometheusExporter::Client.default
-
-      stop if @thread
-
-      @thread = Thread.new do
-        while true
-          begin
-            metrics = active_record_collector.collect
-            metrics.each { |metric| client.send_json metric }
-          rescue => e
-            STDERR.puts("Prometheus Exporter Failed To Collect Process Stats #{e}")
-          ensure
-            sleep frequency
-          end
-        end
+      worker_loop do
+        metrics = active_record_collector.collect
+        metrics.each { |metric| client.send_json metric }
       end
+
+      super
     end
 
     def self.validate_config_labels(config_labels)
@@ -41,13 +32,6 @@ module PrometheusExporter::Instrumentation
       raise "Invalid Config Labels, available options #{ALLOWED_CONFIG_LABELS}" if (config_labels - ALLOWED_CONFIG_LABELS).size > 0
     end
 
-    def self.stop
-      if t = @thread
-        t.kill
-        @thread = nil
-      end
-    end
-
     def initialize(metric_labels, config_labels)
       @metric_labels = metric_labels
       @config_labels = config_labels
@@ -81,15 +65,13 @@ module PrometheusExporter::Instrumentation
     private
 
     def labels(pool)
-      if pool.respond_to?(:spec) # ActiveRecord <= 6.0
+      if ::ActiveRecord.version < Gem::Version.new("6.1.0.rc1")
         @metric_labels.merge(pool_name: pool.spec.name).merge(pool.spec.config
           .select { |k, v| @config_labels.include? k }
           .map { |k, v| [k.to_s.dup.prepend("dbconfig_"), v] }.to_h)
-      elsif pool.respond_to?(:db_config) # ActiveRecord >= 6.1.rc1
+      else
         @metric_labels.merge(pool_name: pool.db_config.name).merge(
           @config_labels.each_with_object({}) { |l, acc| acc["dbconfig_#{l}"] = pool.db_config.public_send(l) })
-      else
-        raise "Unsupported connection pool"
       end
     end
   end

data/lib/prometheus_exporter/instrumentation/delayed_job.rb

@@ -13,8 +13,8 @@ module PrometheusExporter::Instrumentation
           callbacks do |lifecycle|
             lifecycle.around(:invoke_job) do |job, *args, &block|
               max_attempts = Delayed::Worker.max_attempts
-              enqueued_count = Delayed::Job.count
-              pending_count = Delayed::Job.where(attempts: 0, locked_at: nil).count
+              enqueued_count = Delayed::Job.where(queue: job.queue).count
+              pending_count = Delayed::Job.where(attempts: 0, locked_at: nil, queue: job.queue).count
               instrumenter.call(job, max_attempts, enqueued_count, pending_count, *args, &block)
             end
           end
@@ -31,6 +31,7 @@ module PrometheusExporter::Instrumentation
     def call(job, max_attempts, enqueued_count, pending_count, *args, &block)
       success = false
       start = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      latency = Time.current - job.run_at
       attempts = job.attempts + 1 # Increment because we're adding the current attempt
       result = block.call(job, *args)
       success = true
@@ -41,8 +42,10 @@ module PrometheusExporter::Instrumentation
       @client.send_json(
         type: "delayed_job",
         name: job.handler.to_s.match(JOB_CLASS_REGEXP).to_a[1].to_s,
+        queue_name: job.queue,
         success: success,
         duration: duration,
+        latency: latency,
         attempts: attempts,
         max_attempts: max_attempts,
         enqueued: enqueued_count,

data/lib/prometheus_exporter/instrumentation/good_job.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+# collects stats from GoodJob
+module PrometheusExporter::Instrumentation
+  class GoodJob < PeriodicStats
+    def self.start(client: nil, frequency: 30)
+      good_job_collector = new
+      client ||= PrometheusExporter::Client.default
+
+      worker_loop do
+        client.send_json(good_job_collector.collect)
+      end
+
+      super
+    end
+
+    def collect
+      {
+        type: "good_job",
+        scheduled: ::GoodJob::Job.scheduled.size,
+        retried: ::GoodJob::Job.retried.size,
+        queued: ::GoodJob::Job.queued.size,
+        running: ::GoodJob::Job.running.size,
+        finished: ::GoodJob::Job.finished.size,
+        succeeded: ::GoodJob::Job.succeeded.size,
+        discarded: ::GoodJob::Job.discarded.size
+      }
+    end
+  end
+end

data/lib/prometheus_exporter/instrumentation/method_profiler.rb

@@ -4,29 +4,14 @@
 module PrometheusExporter::Instrumentation; end
 
 class PrometheusExporter::Instrumentation::MethodProfiler
-  def self.patch(klass, methods, name)
-    patches = methods.map do |method_name|
-      <<~RUBY
-      unless defined?(#{method_name}__mp_unpatched)
-        alias_method :#{method_name}__mp_unpatched, :#{method_name}
-        def #{method_name}(*args, &blk)
-          unless prof = Thread.current[:_method_profiler]
-            return #{method_name}__mp_unpatched(*args, &blk)
-          end
-          begin
-            start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-            #{method_name}__mp_unpatched(*args, &blk)
-          ensure
-            data = (prof[:#{name}] ||= {duration: 0.0, calls: 0})
-            data[:duration] += Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
-            data[:calls] += 1
-          end
-        end
-      end
-      RUBY
-    end.join("\n")
-
-    klass.class_eval patches
+  def self.patch(klass, methods, name, instrument:)
+    if instrument == :alias_method
+      patch_using_alias_method(klass, methods, name)
+    elsif instrument == :prepend
+      patch_using_prepend(klass, methods, name)
+    else
+      raise ArgumentError, "instrument must be :alias_method or :prepend"
+    end
   end
 
   def self.transfer
@@ -54,4 +39,59 @@ class PrometheusExporter::Instrumentation::MethodProfiler
     end
     data
   end
+
+  def self.define_methods_on_module(klass, methods, name)
+    patch_source_line = __LINE__ + 3
+    patches = methods.map do |method_name|
+      <<~RUBY
+        def #{method_name}(*args, &blk)
+          unless prof = Thread.current[:_method_profiler]
+            return super
+          end
+          begin
+            start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+            super
+          ensure
+            data = (prof[:#{name}] ||= {duration: 0.0, calls: 0})
+            data[:duration] += Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
+            data[:calls] += 1
+          end
+        end
+      RUBY
+    end.join("\n")
+
+    klass.module_eval patches, __FILE__, patch_source_line
+  end
+
+  def self.patch_using_prepend(klass, methods, name)
+    prepend_instrument = Module.new
+    define_methods_on_module(prepend_instrument, methods, name)
+    klass.prepend(prepend_instrument)
+  end
+
+  def self.patch_using_alias_method(klass, methods, name)
+    patch_source_line = __LINE__ + 3
+    patches = methods.map do |method_name|
+      <<~RUBY
+      unless defined?(#{method_name}__mp_unpatched)
+        alias_method :#{method_name}__mp_unpatched, :#{method_name}
+        def #{method_name}(*args, &blk)
+          unless prof = Thread.current[:_method_profiler]
+            return #{method_name}__mp_unpatched(*args, &blk)
+          end
+          begin
+            start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+            #{method_name}__mp_unpatched(*args, &blk)
+          ensure
+            data = (prof[:#{name}] ||= {duration: 0.0, calls: 0})
+            data[:duration] += Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
+            data[:calls] += 1
+          end
+        end
+      end
+      RUBY
+    end.join("\n")
+
+    klass.class_eval patches, __FILE__, patch_source_line
+  end
 end

data/lib/prometheus_exporter/instrumentation/periodic_stats.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module PrometheusExporter::Instrumentation
+  class PeriodicStats
+
+    def self.start(*args, frequency:, client: nil, **kwargs)
+      client ||= PrometheusExporter::Client.default
+
+      if !(Numeric === frequency)
+        raise ArgumentError.new("Expected frequency to be a number")
+      end
+
+      if frequency < 0
+        raise ArgumentError.new("Expected frequency to be a positive number")
+      end
+
+      if !@worker_loop
+        raise ArgumentError.new("Worker loop was not set")
+      end
+
+      klass = self
+
+      stop
+
+      @stop_thread = false
+
+      @thread = Thread.new do
+        while !@stop_thread
+          begin
+            @worker_loop.call
+          rescue => e
+            client.logger.error("#{klass} Prometheus Exporter Failed To Collect Stats #{e}")
+          ensure
+            sleep frequency
+          end
+        end
+      end
+
+    end
+
+    def self.started?
+      !!@thread&.alive?
+    end
+
+    def self.worker_loop(&blk)
+      @worker_loop = blk
+    end
+
+    def self.stop
+      # to avoid a warning
+      @thread = nil if !defined?(@thread)
+
+      if @thread&.alive?
+        @stop_thread = true
+        @thread.wakeup
+        @thread.join
+      end
+      @thread = nil
+    end
+
+  end
+end

data/lib/prometheus_exporter/instrumentation/process.rb

@@ -2,8 +2,7 @@
 
 # collects stats from currently running process
 module PrometheusExporter::Instrumentation
-  class Process
-    @thread = nil if !defined?(@thread)
+  class Process < PeriodicStats
 
     def self.start(client: nil, type: "ruby", frequency: 30, labels: nil)
 
@@ -19,27 +18,12 @@ module PrometheusExporter::Instrumentation
       process_collector = new(metric_labels)
       client ||= PrometheusExporter::Client.default
 
-      stop if @thread
-
-      @thread = Thread.new do
-        while true
-          begin
-            metric = process_collector.collect
-            client.send_json metric
-          rescue => e
-            STDERR.puts("Prometheus Exporter Failed To Collect Process Stats #{e}")
-          ensure
-            sleep frequency
-          end
-        end
+      worker_loop do
+        metric = process_collector.collect
+        client.send_json metric
       end
-    end
 
-    def self.stop
-      if t = @thread
-        t.kill
-        @thread = nil
-      end
+      super
     end
 
     def initialize(metric_labels)

data/lib/prometheus_exporter/instrumentation/puma.rb

@@ -4,39 +4,46 @@ require "json"
 
 # collects stats from puma
 module PrometheusExporter::Instrumentation
-  class Puma
-    def self.start(client: nil, frequency: 30)
-      puma_collector = new
+  class Puma < PeriodicStats
+    def self.start(client: nil, frequency: 30, labels: {})
+      puma_collector = new(labels)
       client ||= PrometheusExporter::Client.default
-      Thread.new do
-        while true
-          begin
-            metric = puma_collector.collect
-            client.send_json metric
-          rescue => e
-            STDERR.puts("Prometheus Exporter Failed To Collect Puma Stats #{e}")
-          ensure
-            sleep frequency
-          end
-        end
+
+      worker_loop do
+        metric = puma_collector.collect
+        client.send_json metric
       end
+
+      super
+    end
+
+    def initialize(metric_labels = {})
+      @metric_labels = metric_labels
     end
 
     def collect
-      metric = {}
-      metric[:type] = "puma"
+      metric = {
+        pid: pid,
+        type: "puma",
+        hostname: ::PrometheusExporter.hostname,
+        metric_labels: @metric_labels
+      }
       collect_puma_stats(metric)
       metric
     end
 
+    def pid
+      @pid = ::Process.pid
+    end
+
     def collect_puma_stats(metric)
       stats = JSON.parse(::Puma.stats)
 
       if stats.key?("workers")
         metric[:phase] = stats["phase"]
-        metric[:workers_total] = stats["workers"]
-        metric[:booted_workers_total] = stats["booted_workers"]
-        metric[:old_workers_total] = stats["old_workers"]
+        metric[:workers] = stats["workers"]
+        metric[:booted_workers] = stats["booted_workers"]
+        metric[:old_workers] = stats["old_workers"]
 
         stats["worker_status"].each do |worker|
           next if worker["last_status"].empty?
@@ -50,15 +57,15 @@ module PrometheusExporter::Instrumentation
     private
 
     def collect_worker_status(metric, status)
-      metric[:request_backlog_total] ||= 0
-      metric[:running_threads_total] ||= 0
-      metric[:thread_pool_capacity_total] ||= 0
-      metric[:max_threads_total] ||= 0
+      metric[:request_backlog] ||= 0
+      metric[:running_threads] ||= 0
+      metric[:thread_pool_capacity] ||= 0
+      metric[:max_threads] ||= 0
 
-      metric[:request_backlog_total] += status["backlog"]
-      metric[:running_threads_total] += status["running"]
-      metric[:thread_pool_capacity_total] += status["pool_capacity"]
-      metric[:max_threads_total] += status["max_threads"]
+      metric[:request_backlog] += status["backlog"]
+      metric[:running_threads] += status["running"]
+      metric[:thread_pool_capacity] += status["pool_capacity"]
+      metric[:max_threads] += status["max_threads"]
     end
   end
 end

data/lib/prometheus_exporter/instrumentation/resque.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+# collects stats from resque
+module PrometheusExporter::Instrumentation
+  class Resque < PeriodicStats
+    def self.start(client: nil, frequency: 30)
+      resque_collector = new
+      client ||= PrometheusExporter::Client.default
+
+      worker_loop do
+        client.send_json(resque_collector.collect)
+      end
+
+      super
+    end
+
+    def collect
+      metric = {}
+      metric[:type] = "resque"
+      collect_resque_stats(metric)
+      metric
+    end
+
+    def collect_resque_stats(metric)
+      info = ::Resque.info
+
+      metric[:processed_jobs] = info[:processed]
+      metric[:failed_jobs] = info[:failed]
+      metric[:pending_jobs] = info[:pending]
+      metric[:queues] = info[:queues]
+      metric[:worker] = info[:workers]
+      metric[:working] = info[:working]
+    end
+  end
+end