prometheus_exporter 0.7.0 → 2.3.0

data/lib/prometheus_exporter/client.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require 'socket'
-require 'thread'
+require "socket"
+require "logger"
 
 module PrometheusExporter
   class Client
@@ -17,14 +17,10 @@ module PrometheusExporter
       end
 
       def standard_values(value, keys, prometheus_exporter_action = nil)
-        values = {
-          type: @type,
-          help: @help,
-          name: @name,
-          keys: keys,
-          value: value
-        }
-        values[:prometheus_exporter_action] = prometheus_exporter_action if prometheus_exporter_action
+        values = { type: @type, help: @help, name: @name, keys: keys, value: value }
+        values[
+          :prometheus_exporter_action
+        ] = prometheus_exporter_action if prometheus_exporter_action
         values[:opts] = @opts if @opts
         values
       end
@@ -53,14 +49,21 @@ module PrometheusExporter
     MAX_SOCKET_AGE = 25
     MAX_QUEUE_SIZE = 10_000
 
+    attr_reader :logger
+
     def initialize(
-      host: ENV.fetch('PROMETHEUS_EXPORTER_HOST', 'localhost'),
-      port: ENV.fetch('PROMETHEUS_EXPORTER_PORT', PrometheusExporter::DEFAULT_PORT),
+      host: ENV.fetch("PROMETHEUS_EXPORTER_HOST", "localhost"),
+      port: ENV.fetch("PROMETHEUS_EXPORTER_PORT", PrometheusExporter::DEFAULT_PORT),
       max_queue_size: nil,
       thread_sleep: 0.5,
       json_serializer: nil,
-      custom_labels: nil
+      custom_labels: nil,
+      logger: Logger.new(STDERR),
+      log_level: Logger::WARN,
+      process_queue_once_and_stop: false
     )
+      @logger = logger
+      @logger.level = log_level
       @metrics = []
 
       @queue = Queue.new
@@ -72,9 +75,7 @@ module PrometheusExporter
       max_queue_size ||= MAX_QUEUE_SIZE
       max_queue_size = max_queue_size.to_i
 
-      if max_queue_size.to_i <= 0
-        raise ArgumentError, "max_queue_size must be larger than 0"
-      end
+      raise ArgumentError, "max_queue_size must be larger than 0" if max_queue_size <= 0
 
       @max_queue_size = max_queue_size
       @host = host
@@ -86,6 +87,7 @@ module PrometheusExporter
       @json_serializer = json_serializer == :oj ? PrometheusExporter::OjCompat : JSON
 
       @custom_labels = custom_labels
+      @process_queue_once_and_stop = process_queue_once_and_stop
     end
 
     def custom_labels=(custom_labels)
@@ -125,7 +127,7 @@ module PrometheusExporter
     def send(str)
       @queue << str
       if @queue.length > @max_queue_size
-        STDERR.puts "Prometheus Exporter client is dropping message cause queue is full"
+        logger.warn "Prometheus Exporter client is dropping message cause queue is full"
         @queue.pop
       end
 
@@ -143,8 +145,8 @@ module PrometheusExporter
           @socket.write(message)
           @socket.write("\r\n")
         rescue => e
-          STDERR.puts "Prometheus Exporter is dropping a message: #{e}"
-          @socket = nil
+          logger.warn "Prometheus Exporter is dropping a message: #{e}"
+          close_socket!
           raise
         end
       end
@@ -154,9 +156,7 @@ module PrometheusExporter
       @mutex.synchronize do
         wait_for_empty_queue_with_timeout(wait_timeout_seconds)
         @worker_thread&.kill
-        while @worker_thread&.alive?
-          sleep 0.001
-        end
+        sleep 0.001 while @worker_thread&.alive?
         @worker_thread = nil
         close_socket!
       end
@@ -168,22 +168,31 @@ module PrometheusExporter
       close_socket_if_old!
       process_queue
     rescue => e
-      STDERR.puts "Prometheus Exporter, failed to send message #{e}"
+      logger.error "Prometheus Exporter, failed to send message #{e}"
     end
 
     def ensure_worker_thread!
+      if @process_queue_once_and_stop
+        worker_loop
+        return
+      end
+
       unless @worker_thread&.alive?
         @mutex.synchronize do
           return if @worker_thread&.alive?
 
-          @worker_thread = Thread.new do
-            while true
-              worker_loop
-              sleep @thread_sleep
+          @worker_thread =
+            Thread.new do
+              while true
+                worker_loop
+                sleep @thread_sleep
+              end
             end
-          end
         end
       end
+    rescue ThreadError => e
+      raise unless e.message =~ /can't alloc thread/
+      logger.error "Prometheus Exporter, failed to send message ThreadError #{e}"
     end
 
     def close_socket!
@@ -202,7 +211,8 @@ module PrometheusExporter
     end
 
     def close_socket_if_old!
-      if @socket_pid == Process.pid && @socket && @socket_started && ((@socket_started + MAX_SOCKET_AGE) < Time.now.to_f)
+      if @socket_pid == Process.pid && @socket && @socket_started &&
+           ((@socket_started + MAX_SOCKET_AGE) < Time.now.to_f)
         close_socket!
       end
     end
@@ -230,9 +240,8 @@ module PrometheusExporter
       end
 
       nil
-    rescue
-      @socket = nil
-      @socket_started = nil
+    rescue StandardError
+      close_socket!
       @socket_pid = nil
       raise
     end

data/lib/prometheus_exporter/instrumentation/active_record.rb

@@ -2,14 +2,17 @@
 
 # collects stats from currently running process
 module PrometheusExporter::Instrumentation
-  class ActiveRecord
-    ALLOWED_CONFIG_LABELS = %i(database username host port)
+  class ActiveRecord < PeriodicStats
+    ALLOWED_CONFIG_LABELS = %i[database username host port]
 
     def self.start(client: nil, frequency: 30, custom_labels: {}, config_labels: [])
+      client ||= PrometheusExporter::Client.default
 
-      # Not all rails versions support coonection pool stats
+      # Not all rails versions support connection pool stats
       unless ::ActiveRecord::Base.connection_pool.respond_to?(:stat)
-        STDERR.puts("ActiveRecord connection pool stats not supported in your rails version")
+        client.logger.error(
+          "ActiveRecord connection pool stats not supported in your rails version",
+        )
         return
       end
 
@@ -18,33 +21,18 @@ module PrometheusExporter::Instrumentation
 
       active_record_collector = new(custom_labels, config_labels)
 
-      client ||= PrometheusExporter::Client.default
-
-      stop if @thread
-
-      @thread = Thread.new do
-        while true
-          begin
-            metrics = active_record_collector.collect
-            metrics.each { |metric| client.send_json metric }
-          rescue => e
-            STDERR.puts("Prometheus Exporter Failed To Collect Process Stats #{e}")
-          ensure
-            sleep frequency
-          end
-        end
+      worker_loop do
+        metrics = active_record_collector.collect
+        metrics.each { |metric| client.send_json metric }
       end
+
+      super
     end
 
     def self.validate_config_labels(config_labels)
       return if config_labels.size == 0
-      raise "Invalid Config Labels, available options #{ALLOWED_CONFIG_LABELS}" if (config_labels - ALLOWED_CONFIG_LABELS).size > 0
-    end
-
-    def self.stop
-      if t = @thread
-        t.kill
-        @thread = nil
+      if (config_labels - ALLOWED_CONFIG_LABELS).size > 0
+        raise "Invalid Config Labels, available options #{ALLOWED_CONFIG_LABELS}"
       end
     end
 
@@ -71,7 +59,7 @@ module PrometheusExporter::Instrumentation
           pid: pid,
           type: "active_record",
           hostname: ::PrometheusExporter.hostname,
-          metric_labels: labels(pool)
+          metric_labels: labels(pool),
         }
         metric.merge!(pool.stat)
         metrics << metric
@@ -81,15 +69,21 @@ module PrometheusExporter::Instrumentation
     private
 
     def labels(pool)
-      if pool.respond_to?(:spec) # ActiveRecord <= 6.0
-        @metric_labels.merge(pool_name: pool.spec.name).merge(pool.spec.config
-          .select { |k, v| @config_labels.include? k }
-          .map { |k, v| [k.to_s.dup.prepend("dbconfig_"), v] }.to_h)
-      elsif pool.respond_to?(:db_config) # ActiveRecord >= 6.1.rc1
-        @metric_labels.merge(pool_name: pool.db_config.name).merge(
-          @config_labels.each_with_object({}) { |l, acc| acc["dbconfig_#{l}"] = pool.db_config.public_send(l) })
+      if ::ActiveRecord.version < Gem::Version.new("6.1.0.rc1")
+        @metric_labels.merge(pool_name: pool.spec.name).merge(
+          pool
+            .spec
+            .config
+            .select { |k, v| @config_labels.include? k }
+            .map { |k, v| [k.to_s.dup.prepend("dbconfig_"), v] }
+            .to_h,
+        )
       else
-        raise "Unsupported connection pool"
+        @metric_labels.merge(pool_name: pool.db_config.name).merge(
+          @config_labels.each_with_object({}) do |l, acc|
+            acc["dbconfig_#{l}"] = pool.db_config.public_send(l)
+          end,
+        )
       end
     end
   end

data/lib/prometheus_exporter/instrumentation/delayed_job.rb

@@ -2,23 +2,33 @@
 
 module PrometheusExporter::Instrumentation
   class DelayedJob
-    JOB_CLASS_REGEXP = %r{job_class: (\w+:{0,2})+}.freeze
+    JOB_CLASS_REGEXP = /job_class: ((\w+:{0,2})+)/.freeze
 
     class << self
-      def register_plugin(client: nil)
+      def register_plugin(client: nil, include_module_name: false)
         instrumenter = self.new(client: client)
         return unless defined?(Delayed::Plugin)
 
-        plugin = Class.new(Delayed::Plugin) do
-          callbacks do |lifecycle|
-            lifecycle.around(:invoke_job) do |job, *args, &block|
-              max_attempts = Delayed::Worker.max_attempts
-              enqueued_count = Delayed::Job.count
-              pending_count = Delayed::Job.where(attempts: 0, locked_at: nil).count
-              instrumenter.call(job, max_attempts, enqueued_count, pending_count, *args, &block)
+        plugin =
+          Class.new(Delayed::Plugin) do
+            callbacks do |lifecycle|
+              lifecycle.around(:invoke_job) do |job, *args, &block|
+                max_attempts = Delayed::Worker.max_attempts
+                enqueued_count = Delayed::Job.where(queue: job.queue).count
+                pending_count =
+                  Delayed::Job.where(attempts: 0, locked_at: nil, queue: job.queue).count
+                instrumenter.call(
+                  job,
+                  max_attempts,
+                  enqueued_count,
+                  pending_count,
+                  include_module_name,
+                  *args,
+                  &block
+                )
+              end
             end
           end
-        end
 
         Delayed::Worker.plugins << plugin
       end
@@ -28,9 +38,12 @@ module PrometheusExporter::Instrumentation
       @client = client || PrometheusExporter::Client.default
     end
 
-    def call(job, max_attempts, enqueued_count, pending_count, *args, &block)
+    def call(job, max_attempts, enqueued_count, pending_count, include_module_name, *args, &block)
       success = false
+      job_name = job.handler.to_s.match(JOB_CLASS_REGEXP).to_a[include_module_name ? 1 : 2].to_s
+      job_name ||= job.try(:name)
       start = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      latency = Time.current - job.run_at
       attempts = job.attempts + 1 # Increment because we're adding the current attempt
       result = block.call(job, *args)
       success = true
@@ -40,13 +53,15 @@ module PrometheusExporter::Instrumentation
 
       @client.send_json(
         type: "delayed_job",
-        name: job.handler.to_s.match(JOB_CLASS_REGEXP).to_a[1].to_s,
+        name: job_name,
+        queue_name: job.queue,
         success: success,
         duration: duration,
+        latency: latency,
         attempts: attempts,
         max_attempts: max_attempts,
         enqueued: enqueued_count,
-        pending: pending_count
+        pending: pending_count,
       )
     end
   end

data/lib/prometheus_exporter/instrumentation/good_job.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+# collects stats from GoodJob
+module PrometheusExporter::Instrumentation
+  class GoodJob < PeriodicStats
+    def self.start(client: nil, frequency: 30)
+      good_job_collector = new
+      client ||= PrometheusExporter::Client.default
+
+      worker_loop { client.send_json(good_job_collector.collect) }
+
+      super
+    end
+
+    def collect
+      {
+        type: "good_job",
+        scheduled: ::GoodJob::Job.scheduled.size,
+        retried: ::GoodJob::Job.retried.size,
+        queued: ::GoodJob::Job.queued.size,
+        running: ::GoodJob::Job.running.size,
+        finished: ::GoodJob::Job.finished.size,
+        succeeded: ::GoodJob::Job.succeeded.size,
+        discarded: ::GoodJob::Job.discarded.size,
+      }
+    end
+  end
+end

data/lib/prometheus_exporter/instrumentation/hutch.rb

@@ -19,7 +19,7 @@ module PrometheusExporter::Instrumentation
         type: "hutch",
         name: @klass.class.to_s,
         success: success,
-        duration: duration
+        duration: duration,
       )
     end
   end

data/lib/prometheus_exporter/instrumentation/method_profiler.rb

@@ -1,32 +1,18 @@
 # frozen_string_literal: true
 
 # see https://samsaffron.com/archive/2017/10/18/fastest-way-to-profile-a-method-in-ruby
-module PrometheusExporter::Instrumentation; end
+module PrometheusExporter::Instrumentation
+end
 
 class PrometheusExporter::Instrumentation::MethodProfiler
-  def self.patch(klass, methods, name)
-    patches = methods.map do |method_name|
-      <<~RUBY
-      unless defined?(#{method_name}__mp_unpatched)
-        alias_method :#{method_name}__mp_unpatched, :#{method_name}
-        def #{method_name}(*args, &blk)
-          unless prof = Thread.current[:_method_profiler]
-            return #{method_name}__mp_unpatched(*args, &blk)
-          end
-          begin
-            start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-            #{method_name}__mp_unpatched(*args, &blk)
-          ensure
-            data = (prof[:#{name}] ||= {duration: 0.0, calls: 0})
-            data[:duration] += Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
-            data[:calls] += 1
-          end
-        end
-      end
-      RUBY
-    end.join("\n")
-
-    klass.class_eval patches
+  def self.patch(klass, methods, name, instrument:)
+    if instrument == :alias_method
+      patch_using_alias_method(klass, methods, name)
+    elsif instrument == :prepend
+      patch_using_prepend(klass, methods, name)
+    else
+      raise ArgumentError, "instrument must be :alias_method or :prepend"
+    end
   end
 
   def self.transfer
@@ -36,9 +22,8 @@ class PrometheusExporter::Instrumentation::MethodProfiler
   end
 
   def self.start(transfer = nil)
-    Thread.current[:_method_profiler] = transfer || {
-      __start: Process.clock_gettime(Process::CLOCK_MONOTONIC)
-    }
+    Thread.current[:_method_profiler] = transfer ||
+      { __start: Process.clock_gettime(Process::CLOCK_MONOTONIC) }
   end
 
   def self.clear
@@ -54,4 +39,59 @@ class PrometheusExporter::Instrumentation::MethodProfiler
     end
     data
   end
+
+  def self.define_methods_on_module(klass, methods, name)
+    patch_source_line = __LINE__ + 3
+
+    patches = methods.map { |method_name| <<~RUBY }.join("\n")
+        def #{method_name}(...)
+          unless prof = Thread.current[:_method_profiler]
+            return super
+          end
+          begin
+            start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+            super
+          ensure
+            data = (prof[:#{name}] ||= {duration: 0.0, calls: 0})
+            data[:duration] += Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
+            data[:calls] += 1
+          end
+        end
+      RUBY
+
+    klass.module_eval(patches, __FILE__, patch_source_line)
+  end
+
+  def self.patch_using_prepend(klass, methods, name)
+    prepend_instrument = Module.new
+    define_methods_on_module(prepend_instrument, methods, name)
+    klass.prepend(prepend_instrument)
+  end
+
+  def self.patch_using_alias_method(klass, methods, name)
+    patch_source_line = __LINE__ + 3
+
+    patches = methods.map { |method_name| <<~RUBY }.join("\n")
+      unless defined?(#{method_name}__mp_unpatched)
+        alias_method :#{method_name}__mp_unpatched, :#{method_name}
+
+        def #{method_name}(...)
+          unless prof = Thread.current[:_method_profiler]
+            return #{method_name}__mp_unpatched(...)
+          end
+
+          begin
+            start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+            #{method_name}__mp_unpatched(...)
+          ensure
+            data = (prof[:#{name}] ||= {duration: 0.0, calls: 0})
+            data[:duration] += Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
+            data[:calls] += 1
+          end
+        end
+      end
+      RUBY
+
+    klass.class_eval(patches, __FILE__, patch_source_line)
+  end
 end

data/lib/prometheus_exporter/instrumentation/periodic_stats.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module PrometheusExporter::Instrumentation
+  class PeriodicStats
+    def self.start(*args, frequency:, client: nil, **kwargs)
+      client ||= PrometheusExporter::Client.default
+
+      raise ArgumentError.new("Expected frequency to be a number") if !(Numeric === frequency)
+
+      raise ArgumentError.new("Expected frequency to be a positive number") if frequency < 0
+
+      raise ArgumentError.new("Worker loop was not set") if !@worker_loop
+
+      klass = self
+
+      stop
+
+      @stop_thread = false
+
+      @thread =
+        Thread.new do
+          while !@stop_thread
+            begin
+              @worker_loop.call
+            rescue => e
+              client.logger.error("#{klass} Prometheus Exporter Failed To Collect Stats #{e}")
+            ensure
+              sleep frequency
+            end
+          end
+        end
+    end
+
+    def self.started?
+      !!@thread&.alive?
+    end
+
+    def self.worker_loop(&blk)
+      @worker_loop = blk
+    end
+
+    def self.stop
+      # to avoid a warning
+      @thread = nil if !defined?(@thread)
+
+      if @thread&.alive?
+        @stop_thread = true
+        @thread.wakeup
+        @thread.join
+      end
+      @thread = nil
+    end
+  end
+end

data/lib/prometheus_exporter/instrumentation/process.rb

@@ -2,11 +2,8 @@
 
 # collects stats from currently running process
 module PrometheusExporter::Instrumentation
-  class Process
-    @thread = nil if !defined?(@thread)
-
+  class Process < PeriodicStats
     def self.start(client: nil, type: "ruby", frequency: 30, labels: nil)
-
       metric_labels =
         if labels && type
           labels.merge(type: type)
@@ -19,27 +16,12 @@ module PrometheusExporter::Instrumentation
       process_collector = new(metric_labels)
       client ||= PrometheusExporter::Client.default
 
-      stop if @thread
-
-      @thread = Thread.new do
-        while true
-          begin
-            metric = process_collector.collect
-            client.send_json metric
-          rescue => e
-            STDERR.puts("Prometheus Exporter Failed To Collect Process Stats #{e}")
-          ensure
-            sleep frequency
-          end
-        end
+      worker_loop do
+        metric = process_collector.collect
+        client.send_json metric
       end
-    end
 
-    def self.stop
-      if t = @thread
-        t.kill
-        @thread = nil
-      end
+      super
     end
 
     def initialize(metric_labels)
@@ -62,16 +44,26 @@ module PrometheusExporter::Instrumentation
     end
 
     def rss
-      @pagesize ||= `getconf PAGESIZE`.to_i rescue 4096
-      File.read("/proc/#{pid}/statm").split(' ')[1].to_i * @pagesize rescue 0
+      @pagesize ||=
+        begin
+          `getconf PAGESIZE`.to_i
+        rescue StandardError
+          4096
+        end
+      begin
+        File.read("/proc/#{pid}/statm").split(" ")[1].to_i * @pagesize
+      rescue StandardError
+        0
+      end
     end
 
     def collect_process_stats(metric)
       metric[:pid] = pid
       metric[:rss] = rss
-
     end
 
+    SWEEPING_AND_MARKING = RUBY_VERSION >= "3.3.0"
+
     def collect_gc_stats(metric)
       stat = GC.stat
       metric[:heap_live_slots] = stat[:heap_live_slots]
@@ -79,10 +71,16 @@ module PrometheusExporter::Instrumentation
       metric[:major_gc_ops_total] = stat[:major_gc_count]
       metric[:minor_gc_ops_total] = stat[:minor_gc_count]
       metric[:allocated_objects_total] = stat[:total_allocated_objects]
+      metric[:malloc_increase_bytes_limit] = stat[:malloc_increase_bytes_limit]
+      metric[:oldmalloc_increase_bytes_limit] = stat[:oldmalloc_increase_bytes_limit]
+      if SWEEPING_AND_MARKING
+        metric[:marking_time] = stat[:marking_time]
+        metric[:sweeping_time] = stat[:sweeping_time]
+      end
     end
 
     def collect_v8_stats(metric)
-      return if !defined? MiniRacer
+      return if !defined?(MiniRacer)
 
       metric[:v8_heap_count] = metric[:v8_heap_size] = 0
       metric[:v8_heap_size] = metric[:v8_physical_size] = 0