sidekiq-prometheus-exporter 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f39ddc3bdda39080a7b6a0b1051fb1dc90ad11be5a970b0c6f955ef50d99af66
-  data.tar.gz: b5abf1281d9d7afdc53a452f3a5eebb480cc6b7f9e631930f02bf9be52acf8c8
+  metadata.gz: 992a82a14617db8daeecd9c7b100f0a96f1503ada37ad80a509121ce940a6e38
+  data.tar.gz: 06c79dfc4a3652acccd1bffdecff30d46987dfc0a728058b6a8d692e70b59904
 SHA512:
-  metadata.gz: 9db770f2ff366515f412235c4cad0abdc0a7e6331420d82e9a6e24446afce711daaa65449350ec1e23eb1128a47131065472d91a6643e1f20628de6249f55009
-  data.tar.gz: 4605aa418c294b245afae7e7201ecf0e558e778acac024f16ce173675ca70db4ae1a7a2858e43bcc26b26c91c10f0361025edf9127bc4c6a4d43515c079c7be6
+  metadata.gz: 9bf9f6c91f5f5596d27bce500312545ae92366b71b0efff62b57883fabf12aa3b02557357b7cff2faa9fec890756722df842471d425b0219df1920cf3ea37490
+  data.tar.gz: e464afd62399b31b5a3970881a8e6ea73a5f0cca4544429689ba699a275b3b5a18e95fdd7093053a85a946c8e555e06603d7ab71f999d72771a93754451ad0d3

data/.rubocop.yml

@@ -128,5 +128,11 @@ RSpec/NestedGroups:
 RSpec/ScatteredSetup:
   Enabled: true
 
+RSpec/SpecFilePathSuffix:
+  Enabled: false
+
+RSpec/SpecFilePathFormat:
+  Enabled: false
+
 Rake/Desc:
   Enabled: false

data/Appraisals

@@ -15,16 +15,19 @@ appraise 'sidekiq-5.x' do
 end
 
 appraise 'sidekiq-6.x' do
+  gem 'base64'
   gem 'redis', '~> 4.1'
   gem 'sidekiq', '~> 6.0'
 end
 
 appraise 'sidekiq-7.x' do
-  gem 'redis', '~> 5.0'
   gem 'sidekiq', '~> 7.0'
 end
 
+appraise 'sidekiq-8.x' do
+  gem 'sidekiq', '~> 8.0'
+end
+
 appraise 'sidekiq-latest' do
-  gem 'redis', '>= 4', github: 'redis/redis-rb'
-  gem 'sidekiq', '>= 7', github: 'mperham/sidekiq'
+  gem 'sidekiq', '>= 8', github: 'mperham/sidekiq'
 end

data/Gemfile

@@ -9,7 +9,7 @@ gem 'appraisal', '~> 2.2'
 gem 'bundler', '~> 2.1'
 gem 'pry', '~> 0.14'
 gem 'pry-byebug', '~> 3.6'
-gem 'rack-test', '~> 1.1'
+gem 'rack-test', '>= 1.1'
 gem 'rake', '~> 13.0'
 gem 'rspec', '~> 3.0'
 gem 'rubocop', '~> 1.22'

data/README.md

@@ -1,6 +1,8 @@
 ![CI status](https://github.com/Strech/sidekiq-prometheus-exporter/workflows/CI/badge.svg)
 [![Maintainability](https://api.codeclimate.com/v1/badges/bb1b30cd7aca8ecc9413/maintainability)](https://codeclimate.com/github/Strech/sidekiq-prometheus-exporter/maintainability)
 
+[Sidekiq Enterprise]: https://sidekiq.org/products/enterprise.html
+
 # Sidekiq Prometheus Exporter
 
 > — Hey! Sidekiq dashboard stats looks like a Prometheus metrics!?
@@ -31,6 +33,7 @@ _(starting Sidekiq `v4.1.0`)_
 | sidekiq_workers                           | gauge   | The number of workers across all the processes                                                         |
 | sidekiq_processes                         | gauge   | The number of processes                                                                                |
 | sidekiq_host_processes                    | gauge   | The number of processes running on the host (labels: `host`, `quiet`)                                  |
+| sidekiq_host_processes_memory_usage_bytes | gauge   | The amount of real memory (resident set) used by the processes running on the host (labels: `host`)    |
 | sidekiq_busy_workers                      | gauge   | The number of workers performing the job                                                               |
 | sidekiq_enqueued_jobs                     | gauge   | The number of enqueued jobs                                                                            |
 | sidekiq_scheduled_jobs                    | gauge   | The number of jobs scheduled for a future execution                                                    |
@@ -42,6 +45,7 @@ _(starting Sidekiq `v4.1.0`)_
 | sidekiq_queue_workers                     | gauge   | The number of workers serving the queue (labels: `name`)                                               |
 | sidekiq_queue_processes                   | gauge   | The number of processes serving the queue (labels: `name`)                                             |
 | sidekiq_queue_busy_workers                | gauge   | The number of workers performing the job for the queue (labels: `name`)                                |
+| sidekiq_leader_lifetime_seconds           | gauge   | The number of seconds since cluster leader has been created (available only on [Sidekiq Enterprise])   |
 
 <details>
   <summary>Click to expand for all available contribs</summary>
@@ -67,7 +71,7 @@ _(starting Sidekiq `v4.1.0`)_
 Add this line to your application's Gemfile:
 
 ```ruby
-gem 'sidekiq-prometheus-exporter', '~> 0.1'
+gem 'sidekiq-prometheus-exporter', '~> 0.3'
 ```
 
 And then execute:
@@ -79,7 +83,7 @@ $ bundle
 Or install it yourself as:
 
 ```console
-$ gem install sidekiq-prometheus-exporter -v '~> 0.1'
+$ gem install sidekiq-prometheus-exporter -v '~> 0.3'
 ```
 
 ## Rack application

data/Rakefile

@@ -1,7 +1,20 @@
+# frozen_string_literal: true
+
 require 'bundler/gem_tasks'
 require 'rspec/core/rake_task'
 
-RSpec::Core::RakeTask.new(:spec)
+RSpec::Core::RakeTask.new(:spec) do |t, args|
+  require 'sidekiq/version'
+  directory =
+    case Gem::Version.new(Sidekiq::VERSION).segments[0]
+    when 7 then 'sidekiq-7.x'
+    when 8 then 'sidekiq-8.x'
+    else 'sidekiq'
+    end
+
+  t.pattern = "spec/#{directory}/**/*_spec.rb"
+  t.rspec_opts = args.to_a.join(' ')
+end
 
 task default: :spec
 

data/lib/sidekiq/prometheus/exporter/standard.rb

@@ -7,17 +7,31 @@ module Sidekiq
   module Prometheus
     module Exporter
       class Standard
-        TEMPLATE = ERB.new(File.read(File.expand_path('templates/standard.erb', __dir__)))
+        UNKNOWN_IDENTITY = 'unknown-identity'.freeze
+        BYTES_IN_KILOBYTE = 1024
+        OMIT_NEWLINES_MODE = '<>'.freeze
+        TEMPLATE = ERB.new(
+          File.read(File.expand_path('templates/standard.erb', __dir__)),
+          trim_mode: OMIT_NEWLINES_MODE
+        )
 
-        QueueStats = Struct.new(:name, :size, :latency)
-        QueueWorkersStats = Struct.new(:total_workers, :busy_workers, :processes)
-        WorkersStats = Struct.new(:total_workers, :by_queue, :by_host)
+        HostStats = Struct.new(:memory_usage, :by_quiet, keyword_init: true)
+        QueueStats = Struct.new(:name, :size, :latency, keyword_init: true)
+        QueueWorkersStats = Struct.new(
+          :total_workers, :busy_workers, :processes, keyword_init: true
+        )
+        WorkersStats = Struct.new(
+          :total_workers, :by_queue, :by_host, :leader_lifetime, keyword_init: true
+        )
 
         def self.available?
           true
         end
 
         def initialize
+          # Version dependent metrics
+          @show_memory_usage = false
+
           @overview_stats = Sidekiq::Stats.new
           @queues_stats = queues_stats
           @workers_stats = workers_stats
@@ -25,46 +39,84 @@ module Sidekiq
         end
 
         def to_s
-          TEMPLATE.result(binding).chomp!
+          TEMPLATE.result(binding)
         end
 
         private
 
         def queues_stats
           Sidekiq::Queue.all.map do |queue|
-            QueueStats.new(queue.name, queue.size, queue.latency)
+            QueueStats.new(name: queue.name, size: queue.size, latency: queue.latency)
           end
         end
 
         def workers_stats
-          workers_stats = WorkersStats.new(0, {}, {})
+          processes = Sidekiq::ProcessSet.new
+          workers_stats = WorkersStats.new(total_workers: 0, by_queue: {}, by_host: {})
 
-          Sidekiq::ProcessSet.new.each_with_object(workers_stats) do |process, stats|
-            stats.total_workers += process['concurrency'].to_i
+          # NOTE: available only on enterprise starting v5.0.1
+          leader_identity =
+            processes.respond_to?(:leader) ? processes.leader : UNKNOWN_IDENTITY
 
-            stats.by_host[process['hostname']] ||= Hash.new(0)
-            stats.by_host[process['hostname']][process['quiet']] += 1
+          processes.each_with_object(workers_stats) do |process, stats|
+            stats.total_workers += process['concurrency'].to_i
 
-            process['queues'].each do |queue|
-              stats.by_queue[queue] ||= QueueWorkersStats.new(0, 0, 0)
-              stats.by_queue[queue].processes += 1
-              stats.by_queue[queue].busy_workers += process['busy'].to_i
-              stats.by_queue[queue].total_workers += process['concurrency'].to_i
+            if process['identity'] == leader_identity
+              stats.leader_lifetime = Time.now.utc.to_i - process['started_at']
             end
+
+            collect_process_stats_by_host(process, stats.by_host)
+            collect_process_stats_by_queue(process, stats.by_queue)
           end
         end
 
+        def collect_process_stats_by_host(process, stats)
+          stats[process['hostname']] ||= HostStats.new(memory_usage: 0, by_quiet: Hash.new(0))
+          stats[process['hostname']].by_quiet[process['quiet']] += 1
+          # NOTE: available only starting v6.2.0
+          @show_memory_usage ||= true if process['rss']
+          stats[process['hostname']].memory_usage += kilobytes_to_bytes(process['rss'].to_i)
+        end
+
+        def collect_process_stats_by_queue(process, stats)
+          process['queues'].each do |queue|
+            stats[queue] ||= QueueWorkersStats.new(
+              total_workers: 0, busy_workers: 0, processes: 0
+            )
+
+            stats[queue].processes += 1
+            stats[queue].busy_workers += process['busy'].to_i
+            stats[queue].total_workers += process['concurrency'].to_i
+          end
+        end
+
+        def kilobytes_to_bytes(kilobytes)
+          kilobytes * BYTES_IN_KILOBYTE
+        end
+
         def max_processing_times
+          return new_max_processing_times if Sidekiq.const_defined?(:Work)
+
           now = Time.now.to_i
 
           Sidekiq::Workers.new
-            .map { |_, _, execution| execution }
-            .group_by { |execution| execution['queue'] }
-            .each_with_object({}) do |(queue, executions), memo|
-              oldest_execution = executions.min_by { |execution| execution['run_at'] }
-              memo[queue] = now - oldest_execution['run_at']
+            .each_with_object({}) { |(_, _, work), memo| (memo[work['queue']] ||= []).push(work) }
+            .transform_values! do |works|
+              oldest_work = works.min_by { |work| work['run_at'] }
+              now - oldest_work['run_at']
             end
         end
+
+        # TODO: Internally Sidekiq::Work stores `run_at` as a timestamp.
+        #       It would be great to use it instead of invoking `Time.at` call.
+        #       See: https://github.com/sidekiq/sidekiq/blob/v8.0.0/lib/sidekiq/api.rb#L1253-L1255
+        def new_max_processing_times
+          now = Time.now
+
+          Sidekiq::Workers.new
+            .each_with_object({}) { |(_, _, work), memo| (memo[work.queue] ||= []).push(work) }
+            .transform_values! { |works| now - works.min_by(&:run_at).run_at }
+        end
       end
     end
   end

data/lib/sidekiq/prometheus/exporter/templates/standard.erb

@@ -16,8 +16,20 @@ sidekiq_processes <%= format('%d', @overview_stats.processes_size) %>
 
 # HELP sidekiq_host_processes The number of processes running on the host.
 # TYPE sidekiq_host_processes gauge
-<% @workers_stats.by_host.each do |host, stats| %><% stats.each do |quiet, count| %>sidekiq_host_processes{host="<%= host %>",quiet="<%= quiet %>"} <%= format('%i', count) %>
-<% end %><% end %>
+<% @workers_stats.by_host.each do |host, stats| %>
+<% stats.by_quiet.each do |quiet, count| %>
+sidekiq_host_processes{host="<%= host %>",quiet="<%= quiet %>"} <%= format('%i', count) %>
+<% end %>
+<% end %>
+
+<% if @show_memory_usage %>
+# HELP sidekiq_host_processes_memory_usage_bytes The real memory (resident set) size of the processes running on the host.
+# TYPE sidekiq_host_processes_memory_usage_bytes gauge
+<% @workers_stats.by_host.each do |host, stats| %>
+sidekiq_host_processes_memory_usage_bytes{host="<%= host %>"} <%= format('%i', stats.memory_usage) %>
+<% end %>
+
+<% end %>
 # HELP sidekiq_busy_workers The number of workers performing the job.
 # TYPE sidekiq_busy_workers gauge
 sidekiq_busy_workers <%= format('%d', @overview_stats.workers_size) %>
@@ -40,25 +52,42 @@ sidekiq_dead_jobs <%= format('%d', @overview_stats.dead_size) %>
 
 # HELP sidekiq_queue_latency_seconds The number of seconds between oldest job being pushed to the queue and current time.
 # TYPE sidekiq_queue_latency_seconds gauge
-<% @queues_stats.each do |queue| %>sidekiq_queue_latency_seconds{name="<%= queue.name %>"} <%= format('%.3f', queue.latency) %>
+<% @queues_stats.each do |queue| %>
+sidekiq_queue_latency_seconds{name="<%= queue.name %>"} <%= format('%.3f', queue.latency) %>
 <% end %>
+
 # HELP sidekiq_queue_enqueued_jobs The number of enqueued jobs in the queue.
 # TYPE sidekiq_queue_enqueued_jobs gauge
-<% @queues_stats.each do |queue| %>sidekiq_queue_enqueued_jobs{name="<%= queue.name %>"} <%= format('%d', queue.size) %>
+<% @queues_stats.each do |queue| %>
+sidekiq_queue_enqueued_jobs{name="<%= queue.name %>"} <%= format('%d', queue.size) %>
 <% end %>
+
 # HELP sidekiq_queue_max_processing_time_seconds The number of seconds between oldest job of the queue being executed and current time.
 # TYPE sidekiq_queue_max_processing_time_seconds gauge
-<% @max_processing_times.each do |queue, max_processing_time| %>sidekiq_queue_max_processing_time_seconds{name="<%= queue %>"} <%= format('%i', max_processing_time) %>
+<% @max_processing_times.each do |queue, max_processing_time| %>
+sidekiq_queue_max_processing_time_seconds{name="<%= queue %>"} <%= format('%i', max_processing_time) %>
 <% end %>
+
 # HELP sidekiq_queue_workers The number of workers serving the queue.
 # TYPE sidekiq_queue_workers gauge
-<% @workers_stats.by_queue.each do |queue, stats| %>sidekiq_queue_workers{name="<%= queue %>"} <%= format('%i', stats.total_workers) %>
+<% @workers_stats.by_queue.each do |queue, stats| %>
+sidekiq_queue_workers{name="<%= queue %>"} <%= format('%i', stats.total_workers) %>
 <% end %>
+
 # HELP sidekiq_queue_processes The number of processes serving the queue.
 # TYPE sidekiq_queue_processes gauge
-<% @workers_stats.by_queue.each do |queue, stats| %>sidekiq_queue_processes{name="<%= queue %>"} <%= format('%i', stats.processes) %>
+<% @workers_stats.by_queue.each do |queue, stats| %>
+sidekiq_queue_processes{name="<%= queue %>"} <%= format('%i', stats.processes) %>
 <% end %>
+
 # HELP sidekiq_queue_busy_workers The number of workers performing the job for the queue.
 # TYPE sidekiq_queue_busy_workers gauge
-<% @workers_stats.by_queue.each do |queue, stats| %>sidekiq_queue_busy_workers{name="<%= queue %>"} <%= format('%i', stats.busy_workers) %>
+<% @workers_stats.by_queue.each do |queue, stats| %>
+sidekiq_queue_busy_workers{name="<%= queue %>"} <%= format('%i', stats.busy_workers) %>
+<% end %>
+<% if @workers_stats.leader_lifetime %>
+
+# HELP sidekiq_leader_lifetime_seconds The number of seconds since the leader process has started.
+# TYPE sidekiq_leader_lifetime_seconds gauge
+sidekiq_leader_lifetime_seconds <%= format('%d', @workers_stats.leader_lifetime) %>
 <% end %>

data/lib/sidekiq/prometheus/exporter/version.rb

@@ -5,7 +5,7 @@ module Sidekiq
     module Exporter
       # NOTE: Every version update dropds Docker patch version to 0
       #       and every adjustment in Docker setup bumps it to +1
-      VERSION = '0.2.1'.freeze
+      VERSION = '0.3.0'.freeze
       DOCKER_PATCH_VERSION = '0'.freeze
     end
   end

data/sidekiq-prometheus-exporter.gemspec

@@ -29,7 +29,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |file| File.basename(file) }
   spec.require_paths = %w(lib)
 
-  spec.required_ruby_version = '>= 2.3'
+  spec.required_ruby_version = '>= 2.5'
   spec.add_dependency 'rack', '>= 1.6.0'
   spec.add_dependency 'sidekiq', '>= 4.1.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-prometheus-exporter
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Sergey Fedorov
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-11-04 00:00:00.000000000 Z
+date: 2025-04-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack
@@ -78,14 +78,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.3'
+      version: '2.5'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.26
+rubygems_version: 3.5.3
 signing_key:
 specification_version: 4
 summary: Prometheus exporter for the Sidekiq