prometheus_exporter 0.7.0 → 2.1.0

data/lib/prometheus_exporter/instrumentation/sidekiq.rb

@@ -1,32 +1,51 @@
 # frozen_string_literal: true
 
-require 'yaml'
+require "yaml"
 
 module PrometheusExporter::Instrumentation
-  JOB_WRAPPER_CLASS_NAME = 'ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper'
-  DELAYED_CLASS_NAMES = [
-    'Sidekiq::Extensions::DelayedClass',
-    'Sidekiq::Extensions::DelayedModel',
-    'Sidekiq::Extensions::DelayedMailer',
+  JOB_WRAPPER_CLASS_NAME =
+    "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
+  DELAYED_CLASS_NAMES = %w[
+    Sidekiq::Extensions::DelayedClass
+    Sidekiq::Extensions::DelayedModel
+    Sidekiq::Extensions::DelayedMailer
   ]
 
   class Sidekiq
     def self.death_handler
-      -> (job, ex) do
+      ->(job, ex) do
         job_is_fire_and_forget = job["retry"] == false
 
+        worker_class = Object.const_get(job["class"])
+        worker_custom_labels = self.get_worker_custom_labels(worker_class, job)
+
         unless job_is_fire_and_forget
           PrometheusExporter::Client.default.send_json(
             type: "sidekiq",
-            name: job["class"],
+            name: get_name(job["class"], job),
             dead: true,
+            custom_labels: worker_custom_labels
           )
         end
       end
     end
 
-    def initialize(client: nil)
-      @client = client || PrometheusExporter::Client.default
+    def self.get_worker_custom_labels(worker_class, msg)
+      return {} unless worker_class.respond_to?(:custom_labels)
+
+      # TODO remove when version 3.0.0 is released
+      method_arity = worker_class.method(:custom_labels).arity
+
+      if method_arity > 0
+        worker_class.custom_labels(msg)
+      else
+        worker_class.custom_labels
+      end
+    end
+
+    def initialize(options = { client: nil })
+      @client =
+        options.fetch(:client, nil) || PrometheusExporter::Client.default
     end
 
     def call(worker, msg, queue)
@@ -43,18 +62,18 @@ module PrometheusExporter::Instrumentation
       duration = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - start
       @client.send_json(
         type: "sidekiq",
-        name: get_name(worker, msg),
+        name: self.class.get_name(worker.class.to_s, msg),
         queue: queue,
         success: success,
         shutdown: shutdown,
-        duration: duration
+        duration: duration,
+        custom_labels: self.class.get_worker_custom_labels(worker.class, msg)
       )
     end
 
     private
 
-    def get_name(worker, msg)
-      class_name = worker.class.to_s
+    def self.get_name(class_name, msg)
       if class_name == JOB_WRAPPER_CLASS_NAME
         get_job_wrapper_name(msg)
       elsif DELAYED_CLASS_NAMES.include?(class_name)
@@ -64,24 +83,35 @@ module PrometheusExporter::Instrumentation
       end
     end
 
-    def get_job_wrapper_name(msg)
-      msg['wrapped']
+    def self.get_job_wrapper_name(msg)
+      msg["wrapped"]
     end
 
-    def get_delayed_name(msg, class_name)
-      # fallback to class_name since we're relying on the internal implementation
-      # of the delayed extensions
-      # https://github.com/mperham/sidekiq/blob/master/lib/sidekiq/extensions/class_methods.rb
+    def self.get_delayed_name(msg, class_name)
       begin
-        (target, method_name, _args) = YAML.load(msg['args'].first)
+        # fallback to class_name since we're relying on the internal implementation
+        # of the delayed extensions
+        # https://github.com/mperham/sidekiq/blob/master/lib/sidekiq/extensions/class_methods.rb
+        target, method_name, _args = YAML.load(msg["args"].first)
         if target.class == Class
           "#{target.name}##{method_name}"
         else
           "#{target.class.name}##{method_name}"
         end
-      rescue
-        class_name
+      rescue Psych::DisallowedClass, ArgumentError
+        parsed = Psych.parse(msg["args"].first)
+        children = parsed.root.children
+        target = (children[0].value || children[0].tag).sub("!", "")
+        method_name = (children[1].value || children[1].tag).sub(":", "")
+
+        if target && method_name
+          "#{target}##{method_name}"
+        else
+          class_name
+        end
       end
+    rescue StandardError
+      class_name
     end
   end
 end

data/lib/prometheus_exporter/instrumentation/sidekiq_process.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module PrometheusExporter::Instrumentation
+  class SidekiqProcess < PeriodicStats
+    def self.start(client: nil, frequency: 30)
+      client ||= PrometheusExporter::Client.default
+      sidekiq_process_collector = new
+
+      worker_loop do
+        client.send_json(sidekiq_process_collector.collect)
+      end
+
+      super
+    end
+
+    def initialize
+      @pid = ::Process.pid
+      @hostname = Socket.gethostname
+    end
+
+    def collect
+      {
+        type: 'sidekiq_process',
+        process: collect_stats
+      }
+    end
+
+    def collect_stats
+      process = current_process
+      return {} unless process
+
+      {
+        busy: process['busy'],
+        concurrency: process['concurrency'],
+        labels: {
+          labels: process['labels'].sort.join(','),
+          queues: process['queues'].sort.join(','),
+          quiet: process['quiet'],
+          tag: process['tag'],
+          hostname: process['hostname'],
+          identity: process['identity'],
+        }
+      }
+    end
+
+    def current_process
+      ::Sidekiq::ProcessSet.new.find do |sp|
+        sp['hostname'] == @hostname && sp['pid'] == @pid
+      end
+    end
+  end
+end

data/lib/prometheus_exporter/instrumentation/sidekiq_queue.rb

@@ -1,22 +1,22 @@
 # frozen_string_literal: true
 
 module PrometheusExporter::Instrumentation
-  class SidekiqQueue
-    def self.start(client: nil, frequency: 30)
+  class SidekiqQueue < PeriodicStats
+    def self.start(client: nil, frequency: 30, all_queues: false)
       client ||= PrometheusExporter::Client.default
-      sidekiq_queue_collector = new
-
-      Thread.new do
-        loop do
-          begin
-            client.send_json(sidekiq_queue_collector.collect)
-          rescue StandardError => e
-            STDERR.puts("Prometheus Exporter Failed To Collect Sidekiq Queue metrics #{e}")
-          ensure
-            sleep frequency
-          end
-        end
+      sidekiq_queue_collector = new(all_queues: all_queues)
+
+      worker_loop do
+        client.send_json(sidekiq_queue_collector.collect)
       end
+
+      super
+    end
+
+    def initialize(all_queues: false)
+      @all_queues = all_queues
+      @pid = ::Process.pid
+      @hostname = Socket.gethostname
     end
 
     def collect
@@ -27,24 +27,32 @@ module PrometheusExporter::Instrumentation
     end
 
     def collect_queue_stats
-      hostname = Socket.gethostname
-      pid = ::Process.pid
-      ps = ::Sidekiq::ProcessSet.new
+      sidekiq_queues = ::Sidekiq::Queue.all
 
-      process = ps.find do |sp|
-        sp['hostname'] == hostname && sp['pid'] == pid
+      unless @all_queues
+        queues = collect_current_process_queues
+        sidekiq_queues.select! { |sidekiq_queue| queues.include?(sidekiq_queue.name) }
       end
 
-      queues = process.nil? ? [] : process['queues']
-
-      ::Sidekiq::Queue.all.map do |queue|
-        next unless queues.include? queue.name
+      sidekiq_queues.map do |queue|
         {
-          backlog_total: queue.size,
+          backlog: queue.size,
           latency_seconds: queue.latency.to_i,
           labels: { queue: queue.name }
         }
       end.compact
     end
+
+    private
+
+    def collect_current_process_queues
+      ps = ::Sidekiq::ProcessSet.new
+
+      process = ps.find do |sp|
+        sp['hostname'] == @hostname && sp['pid'] == @pid
+      end
+
+      process.nil? ? [] : process['queues']
+    end
   end
 end

data/lib/prometheus_exporter/instrumentation/sidekiq_stats.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module PrometheusExporter::Instrumentation
+  class SidekiqStats < PeriodicStats
+    def self.start(client: nil, frequency: 30)
+      client ||= PrometheusExporter::Client.default
+      sidekiq_stats_collector = new
+
+      worker_loop do
+        client.send_json(sidekiq_stats_collector.collect)
+      end
+
+      super
+    end
+
+    def collect
+      {
+        type: 'sidekiq_stats',
+        stats: collect_stats
+      }
+    end
+
+    def collect_stats
+      stats = ::Sidekiq::Stats.new
+      {
+        'dead_size' => stats.dead_size,
+        'enqueued' => stats.enqueued,
+        'failed' => stats.failed,
+        'processed' => stats.processed,
+        'processes_size' => stats.processes_size,
+        'retry_size' => stats.retry_size,
+        'scheduled_size' => stats.scheduled_size,
+        'workers_size' => stats.workers_size,
+      }
+    end
+  end
+end

data/lib/prometheus_exporter/instrumentation/unicorn.rb

@@ -8,22 +8,17 @@ end
 
 module PrometheusExporter::Instrumentation
   # collects stats from unicorn
-  class Unicorn
+  class Unicorn < PeriodicStats
     def self.start(pid_file:, listener_address:, client: nil, frequency: 30)
       unicorn_collector = new(pid_file: pid_file, listener_address: listener_address)
       client ||= PrometheusExporter::Client.default
-      Thread.new do
-        loop do
-          begin
-            metric = unicorn_collector.collect
-            client.send_json metric
-          rescue StandardError => e
-            STDERR.puts("Prometheus Exporter Failed To Collect Unicorn Stats #{e}")
-          ensure
-            sleep frequency
-          end
-        end
+
+      worker_loop do
+        metric = unicorn_collector.collect
+        client.send_json metric
       end
+
+      super
     end
 
     def initialize(pid_file:, listener_address:)
@@ -42,9 +37,9 @@ module PrometheusExporter::Instrumentation
     def collect_unicorn_stats(metric)
       stats = listener_address_stats
 
-      metric[:active_workers_total] = stats.active
-      metric[:request_backlog_total] = stats.queued
-      metric[:workers_total] = worker_process_count
+      metric[:active_workers] = stats.active
+      metric[:request_backlog] = stats.queued
+      metric[:workers] = worker_process_count
     end
 
     private

data/lib/prometheus_exporter/instrumentation.rb

@@ -1,13 +1,18 @@
 # frozen_string_literal: true
 
 require_relative "client"
+require_relative "instrumentation/periodic_stats"
 require_relative "instrumentation/process"
 require_relative "instrumentation/method_profiler"
 require_relative "instrumentation/sidekiq"
 require_relative "instrumentation/sidekiq_queue"
+require_relative "instrumentation/sidekiq_process"
+require_relative "instrumentation/sidekiq_stats"
 require_relative "instrumentation/delayed_job"
 require_relative "instrumentation/puma"
 require_relative "instrumentation/hutch"
 require_relative "instrumentation/unicorn"
 require_relative "instrumentation/active_record"
 require_relative "instrumentation/shoryuken"
+require_relative "instrumentation/resque"
+require_relative "instrumentation/good_job"

data/lib/prometheus_exporter/metric/base.rb

@@ -5,6 +5,7 @@ module PrometheusExporter::Metric
 
     @default_prefix = nil if !defined?(@default_prefix)
     @default_labels = nil if !defined?(@default_labels)
+    @default_aggregation = nil if !defined?(@default_aggregation)
 
     # prefix applied to all metrics
     def self.default_prefix=(name)
@@ -23,6 +24,14 @@ module PrometheusExporter::Metric
       @default_labels || {}
     end
 
+    def self.default_aggregation=(aggregation)
+      @default_aggregation = aggregation
+    end
+
+    def self.default_aggregation
+      @default_aggregation ||= Summary
+    end
+
     attr_accessor :help, :name, :data
 
     def initialize(name, help)
@@ -66,7 +75,7 @@ module PrometheusExporter::Metric
     end
 
     def labels_text(labels)
-      labels = (labels || {}).merge(Base.default_labels)
+      labels = Base.default_labels.merge(labels || {})
       if labels && labels.length > 0
         s = labels.map do |key, value|
           value = value.to_s
@@ -97,15 +106,8 @@ module PrometheusExporter::Metric
       end
     end
 
-    # when we drop Ruby 2.3 we can drop this
-    if "".respond_to? :match?
-      def needs_escape?(str)
-        str.match?(/[\n"\\]/m)
-      end
-    else
-      def needs_escape?(str)
-        !!str.match(/[\n"\\]/m)
-      end
+    def needs_escape?(str)
+      str.match?(/[\n"\\]/m)
     end
 
   end

data/lib/prometheus_exporter/metric/gauge.rb

@@ -5,6 +5,10 @@ module PrometheusExporter::Metric
     attr_reader :data
 
     def initialize(name, help)
+      if name.end_with?("_total")
+        raise ArgumentError, "The metric name of gauge must not have _total suffix. Given: #{name}"
+      end
+
       super
       reset!
     end

data/lib/prometheus_exporter/metric/histogram.rb

@@ -5,9 +5,21 @@ module PrometheusExporter::Metric
 
     DEFAULT_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5.0, 10.0].freeze
 
+    @default_buckets = nil if !defined?(@default_buckets)
+
+    def self.default_buckets
+      @default_buckets || DEFAULT_BUCKETS
+    end
+
+    def self.default_buckets=(buckets)
+      @default_buckets = buckets
+    end
+
+    attr_reader :buckets
+
     def initialize(name, help, opts = {})
       super(name, help)
-      @buckets = (opts[:buckets] || DEFAULT_BUCKETS).sort.reverse
+      @buckets = (opts[:buckets] || self.class.default_buckets).sort
       reset!
     end
 
@@ -45,11 +57,11 @@ module PrometheusExporter::Metric
         first = false
         count = @counts[labels]
         sum = @sums[labels]
-        text << "#{prefix(@name)}_bucket#{labels_text(with_bucket(labels, "+Inf"))} #{count}\n"
         @buckets.each do |bucket|
           value = @observations[labels][bucket]
           text << "#{prefix(@name)}_bucket#{labels_text(with_bucket(labels, bucket.to_s))} #{value}\n"
         end
+        text << "#{prefix(@name)}_bucket#{labels_text(with_bucket(labels, "+Inf"))} #{count}\n"
         text << "#{prefix(@name)}_count#{labels_text(labels)} #{count}\n"
         text << "#{prefix(@name)}_sum#{labels_text(labels)} #{sum}"
       end
@@ -79,7 +91,7 @@ module PrometheusExporter::Metric
     end
 
     def fill_buckets(value, buckets)
-      @buckets.each do |b|
+      @buckets.reverse_each do |b|
         break if value > b
         buckets[b] += 1
       end

data/lib/prometheus_exporter/middleware.rb

@@ -6,23 +6,30 @@ require 'prometheus_exporter/client'
 class PrometheusExporter::Middleware
   MethodProfiler = PrometheusExporter::Instrumentation::MethodProfiler
 
-  def initialize(app, config = { instrument: true, client: nil })
+  def initialize(app, config = { instrument: :alias_method, client: nil })
     @app = app
     @client = config[:client] || PrometheusExporter::Client.default
 
     if config[:instrument]
-      if defined? Redis::Client
-        MethodProfiler.patch(Redis::Client, [:call, :call_pipeline], :redis)
+      if defined?(RedisClient)
+        apply_redis_client_middleware!
+      end
+      if defined?(Redis::VERSION) && (Gem::Version.new(Redis::VERSION) >= Gem::Version.new('5.0.0'))
+        # redis 5 support handled via RedisClient
+      elsif defined? Redis::Client
+        MethodProfiler.patch(Redis::Client, [
+          :call, :call_pipeline
+        ], :redis, instrument: config[:instrument])
       end
       if defined? PG::Connection
         MethodProfiler.patch(PG::Connection, [
           :exec, :async_exec, :exec_prepared, :send_query_prepared, :query
-        ], :sql)
+        ], :sql, instrument: config[:instrument])
       end
       if defined? Mysql2::Client
-        MethodProfiler.patch(Mysql2::Client, [:query], :sql)
-        MethodProfiler.patch(Mysql2::Statement, [:execute], :sql)
-        MethodProfiler.patch(Mysql2::Result, [:each], :sql)
+        MethodProfiler.patch(Mysql2::Client, [:query], :sql, instrument: config[:instrument])
+        MethodProfiler.patch(Mysql2::Statement, [:execute], :sql, instrument: config[:instrument])
+        MethodProfiler.patch(Mysql2::Result, [:each], :sql, instrument: config[:instrument])
       end
     end
   end
@@ -36,11 +43,12 @@ class PrometheusExporter::Middleware
 
     result
   ensure
-
+    status = (result && result[0]) || -1
     obj = {
       type: "web",
       timings: info,
       queue_time: queue_time,
+      status: status,
       default_labels: default_labels(env, result)
     }
     labels = custom_labels(env)
@@ -52,18 +60,21 @@ class PrometheusExporter::Middleware
   end
 
   def default_labels(env, result)
-    status = (result && result[0]) || -1
     params = env["action_dispatch.request.parameters"]
     action = controller = nil
     if params
       action = params["action"]
       controller = params["controller"]
+    elsif (cors = env["rack.cors"]) && cors.respond_to?(:preflight?) && cors.preflight?
+      # if the Rack CORS Middleware identifies the request as a preflight request,
+      # the stack doesn't get to the point where controllers/actions are defined
+      action = "preflight"
+      controller = "preflight"
     end
 
     {
       action: action || "other",
-      controller: controller || "other",
-      status: status
+      controller: controller || "other"
     }
   end
 
@@ -90,19 +101,34 @@ class PrometheusExporter::Middleware
     Process.clock_gettime(Process::CLOCK_REALTIME)
   end
 
-  # get the content of the x-queue-start or x-request-start header
+  # determine queue start from well-known trace headers
   def queue_start(env)
+
+    # get the content of the x-queue-start or x-request-start header
     value = env['HTTP_X_REQUEST_START'] || env['HTTP_X_QUEUE_START']
     unless value.nil? || value == ''
-      convert_header_to_ms(value.to_s)
+      # nginx returns time as milliseconds with 3 decimal places
+      # apache returns time as microseconds without decimal places
+      # this method takes care to convert both into a proper second + fractions timestamp
+      value = value.to_s.gsub(/t=|\./, '')
+      return "#{value[0, 10]}.#{value[10, 13]}".to_f
     end
+
+    # get the content of the x-amzn-trace-id header
+    # see also: https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-request-tracing.html
+    value = env['HTTP_X_AMZN_TRACE_ID']
+    value&.split('Root=')&.last&.split('-')&.fetch(1)&.to_i(16)
+
+  end
+
+  private
+
+  module RedisInstrumenter
+    MethodProfiler.define_methods_on_module(self, ["call", "call_pipelined"], "redis")
   end
 
-  # nginx returns time as milliseconds with 3 decimal places
-  # apache returns time as microseconds without decimal places
-  # this method takes care to convert both into a proper second + fractions timestamp
-  def convert_header_to_ms(str)
-    str = str.gsub(/t=|\./, '')
-    "#{str[0, 10]}.#{str[10, 13]}".to_f
+  def apply_redis_client_middleware!
+    RedisClient.register(RedisInstrumenter)
   end
+
 end

data/lib/prometheus_exporter/server/active_record_collector.rb

@@ -2,7 +2,8 @@
 
 module PrometheusExporter::Server
   class ActiveRecordCollector < TypeCollector
-    MAX_ACTIVERECORD_METRIC_AGE = 60
+    MAX_METRIC_AGE = 60
+
     ACTIVE_RECORD_GAUGES = {
       connections: "Total connections in pool",
       busy: "Connections in use in pool",
@@ -13,7 +14,12 @@ module PrometheusExporter::Server
     }
 
     def initialize
-      @active_record_metrics = []
+      @active_record_metrics = MetricsContainer.new(ttl: MAX_METRIC_AGE)
+      @active_record_metrics.filter = -> (new_metric, old_metric) do
+        new_metric["pid"] == old_metric["pid"] &&
+        new_metric["hostname"] == old_metric["hostname"] &&
+        new_metric["metric_labels"]["pool_name"] == old_metric["metric_labels"]["pool_name"]
+      end
     end
 
     def type
@@ -26,7 +32,7 @@ module PrometheusExporter::Server
       metrics = {}
 
       @active_record_metrics.map do |m|
-        metric_key = (m["metric_labels"] || {}).merge("pid" => m["pid"])
+        metric_key = (m["metric_labels"] || {}).merge("pid" => m["pid"], "hostname" => m["hostname"])
         metric_key.merge!(m["custom_labels"]) if m["custom_labels"]
 
         ACTIVE_RECORD_GAUGES.map do |k, help|
@@ -42,15 +48,6 @@ module PrometheusExporter::Server
     end
 
     def collect(obj)
-      now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
-
-      obj["created_at"] = now
-
-      @active_record_metrics.delete_if do |current|
-        (obj["pid"] == current["pid"] && obj["hostname"] == current["hostname"]) ||
-          (current["created_at"] + MAX_ACTIVERECORD_METRIC_AGE < now)
-      end
-
       @active_record_metrics << obj
     end
   end

data/lib/prometheus_exporter/server/collector.rb

@@ -14,12 +14,16 @@ module PrometheusExporter::Server
       register_collector(ProcessCollector.new)
       register_collector(SidekiqCollector.new)
       register_collector(SidekiqQueueCollector.new)
+      register_collector(SidekiqProcessCollector.new)
+      register_collector(SidekiqStatsCollector.new)
       register_collector(DelayedJobCollector.new)
       register_collector(PumaCollector.new)
       register_collector(HutchCollector.new)
       register_collector(UnicornCollector.new)
       register_collector(ActiveRecordCollector.new)
       register_collector(ShoryukenCollector.new)
+      register_collector(ResqueCollector.new)
+      register_collector(GoodJobCollector.new)
     end
 
     def register_collector(collector)