prometheus_exporter 0.7.0 → 1.0.1

data/lib/prometheus_exporter/server/delayed_job_collector.rb

@@ -19,21 +19,21 @@ module PrometheusExporter::Server
     end
 
     def collect(obj)
-      default_labels = { job_name: obj['name'] }
-      custom_labels = obj['custom_labels']
-      labels = custom_labels.nil? ? default_labels : default_labels.merge(custom_labels)
+      custom_labels = obj['custom_labels'] || {}
+      gauge_labels = { queue_name: obj['queue_name'] }.merge(custom_labels)
+      counter_labels = gauge_labels.merge(job_name: obj['name'])
 
       ensure_delayed_job_metrics
-      @delayed_job_duration_seconds.observe(obj["duration"], labels)
-      @delayed_jobs_total.observe(1, labels)
-      @delayed_failed_jobs_total.observe(1, labels) if !obj["success"]
-      @delayed_jobs_max_attempts_reached_total.observe(1) if obj["attempts"] >= obj["max_attempts"]
-      @delayed_job_duration_seconds_summary.observe(obj["duration"])
-      @delayed_job_duration_seconds_summary.observe(obj["duration"], status: "success") if obj["success"]
-      @delayed_job_duration_seconds_summary.observe(obj["duration"], status: "failed")  if !obj["success"]
-      @delayed_job_attempts_summary.observe(obj["attempts"]) if obj["success"]
-      @delayed_jobs_enqueued.observe(obj["enqueued"])
-      @delayed_jobs_pending.observe(obj["pending"])
+      @delayed_job_duration_seconds.observe(obj["duration"], counter_labels)
+      @delayed_jobs_total.observe(1, counter_labels)
+      @delayed_failed_jobs_total.observe(1, counter_labels) if !obj["success"]
+      @delayed_jobs_max_attempts_reached_total.observe(1, counter_labels) if obj["attempts"] >= obj["max_attempts"]
+      @delayed_job_duration_seconds_summary.observe(obj["duration"], counter_labels)
+      @delayed_job_duration_seconds_summary.observe(obj["duration"], counter_labels.merge(status: "success")) if obj["success"]
+      @delayed_job_duration_seconds_summary.observe(obj["duration"], counter_labels.merge(status: "failed"))  if !obj["success"]
+      @delayed_job_attempts_summary.observe(obj["attempts"], counter_labels) if obj["success"]
+      @delayed_jobs_enqueued.observe(obj["enqueued"], gauge_labels)
+      @delayed_jobs_pending.observe(obj["pending"], gauge_labels)
     end
 
     def metrics
@@ -76,12 +76,12 @@ module PrometheusExporter::Server
                 "delayed_jobs_max_attempts_reached_total", "Total number of delayed jobs that reached max attempts.")
 
         @delayed_job_duration_seconds_summary =
-            PrometheusExporter::Metric::Summary.new("delayed_job_duration_seconds_summary",
-                                                    "Summary of the time it takes jobs to execute.")
+            PrometheusExporter::Metric::Base.default_aggregation.new("delayed_job_duration_seconds_summary",
+                                                                     "Summary of the time it takes jobs to execute.")
 
         @delayed_job_attempts_summary =
-            PrometheusExporter::Metric::Summary.new("delayed_job_attempts_summary",
-                                                    "Summary of the amount of attempts it takes delayed jobs to succeed.")
+            PrometheusExporter::Metric::Base.default_aggregation.new("delayed_job_attempts_summary",
+                                                                     "Summary of the amount of attempts it takes delayed jobs to succeed.")
       end
     end
   end

data/lib/prometheus_exporter/server/puma_collector.rb

@@ -4,13 +4,13 @@ module PrometheusExporter::Server
   class PumaCollector < TypeCollector
     MAX_PUMA_METRIC_AGE = 30
     PUMA_GAUGES = {
-      workers_total: "Number of puma workers.",
-      booted_workers_total: "Number of puma workers booted.",
-      old_workers_total: "Number of old puma workers.",
-      running_threads_total: "Number of puma threads currently running.",
-      request_backlog_total: "Number of requests waiting to be processed by a puma thread.",
-      thread_pool_capacity_total: "Number of puma threads available at current scale.",
-      max_threads_total: "Number of puma threads at available at max scale.",
+      workers: "Number of puma workers.",
+      booted_workers: "Number of puma workers booted.",
+      old_workers: "Number of old puma workers.",
+      running_threads: "Number of puma threads currently running.",
+      request_backlog: "Number of requests waiting to be processed by a puma thread.",
+      thread_pool_capacity: "Number of puma threads available at current scale.",
+      max_threads: "Number of puma threads at available at max scale.",
     }
 
     def initialize
@@ -34,6 +34,9 @@ module PrometheusExporter::Server
         if m["custom_labels"]
           labels.merge!(m["custom_labels"])
         end
+        if m["metric_labels"]
+          labels.merge!(m["metric_labels"])
+        end
 
         PUMA_GAUGES.map do |k, help|
           k = k.to_s
@@ -51,7 +54,12 @@ module PrometheusExporter::Server
       now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
 
       obj["created_at"] = now
-      @puma_metrics.delete_if { |m| m["created_at"] + MAX_PUMA_METRIC_AGE < now }
+
+      @puma_metrics.delete_if do |current|
+        (obj["pid"] == current["pid"] && obj["hostname"] == current["hostname"]) ||
+          (current["created_at"] + MAX_PUMA_METRIC_AGE < now)
+      end
+
       @puma_metrics << obj
     end
   end

data/lib/prometheus_exporter/server/resque_collector.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module PrometheusExporter::Server
+  class ResqueCollector < TypeCollector
+    MAX_RESQUE_METRIC_AGE = 30
+    RESQUE_GAUGES = {
+      processed_jobs: "Total number of processed Resque jobs.",
+      failed_jobs: "Total number of failed Resque jobs.",
+      pending_jobs: "Total number of pending Resque jobs.",
+      queues: "Total number of Resque queues.",
+      workers: "Total number of Resque workers running.",
+      working: "Total number of Resque workers working."
+    }
+
+    def initialize
+      @resque_metrics = []
+      @gauges = {}
+    end
+
+    def type
+      "resque"
+    end
+
+    def metrics
+      return [] if resque_metrics.length == 0
+
+      resque_metrics.map do |metric|
+        labels = metric.fetch("custom_labels", {})
+
+        RESQUE_GAUGES.map do |name, help|
+          name = name.to_s
+          if value = metric[name]
+            gauge = gauges[name] ||= PrometheusExporter::Metric::Gauge.new("resque_#{name}", help)
+            gauge.observe(value, labels)
+          end
+        end
+      end
+
+      gauges.values
+    end
+
+    def collect(object)
+      now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+
+      object["created_at"] = now
+      resque_metrics.delete_if { |metric| metric["created_at"] + MAX_RESQUE_METRIC_AGE < now }
+      resque_metrics << object
+    end
+
+    private
+
+    attr_reader :resque_metrics, :gauges
+  end
+end

data/lib/prometheus_exporter/server/runner.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'prometheus_exporter/client'
+require_relative '../client'
 require_relative '../instrumentation/unicorn'
 
 module PrometheusExporter::Server
@@ -17,6 +17,7 @@ module PrometheusExporter::Server
       @prefix = nil
       @auth = nil
       @realm = nil
+      @histogram = nil
 
       options.each do |k, v|
         send("#{k}=", v) if self.class.method_defined?("#{k}=")
@@ -27,6 +28,10 @@ module PrometheusExporter::Server
       PrometheusExporter::Metric::Base.default_prefix = prefix
       PrometheusExporter::Metric::Base.default_labels = label
 
+      if histogram
+        PrometheusExporter::Metric::Base.default_aggregation = PrometheusExporter::Metric::Histogram
+      end
+
       register_type_collectors
 
       unless collector.is_a?(PrometheusExporter::Server::CollectorBase)
@@ -47,7 +52,7 @@ module PrometheusExporter::Server
     end
 
     attr_accessor :unicorn_listen_address, :unicorn_pid_file
-    attr_writer :prefix, :port, :bind, :collector_class, :type_collectors, :timeout, :verbose, :server_class, :label, :auth, :realm
+    attr_writer :prefix, :port, :bind, :collector_class, :type_collectors, :timeout, :verbose, :server_class, :label, :auth, :realm, :histogram
 
     def auth
       @auth || nil
@@ -98,6 +103,10 @@ module PrometheusExporter::Server
       @label ||= PrometheusExporter::DEFAULT_LABEL
     end
 
+    def histogram
+      @histogram || false
+    end
+
     private
 
     def register_type_collectors

data/lib/prometheus_exporter/server/sidekiq_collector.rb

@@ -52,7 +52,7 @@ module PrometheusExporter::Server
       if !@sidekiq_jobs_total
 
         @sidekiq_job_duration_seconds =
-        PrometheusExporter::Metric::Summary.new(
+        PrometheusExporter::Metric::Base.default_aggregation.new(
           "sidekiq_job_duration_seconds", "Total time spent in sidekiq jobs.")
 
         @sidekiq_jobs_total =

data/lib/prometheus_exporter/server/sidekiq_process_collector.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module PrometheusExporter::Server
+  class SidekiqProcessCollector < PrometheusExporter::Server::TypeCollector
+    MAX_SIDEKIQ_METRIC_AGE = 60
+
+    SIDEKIQ_PROCESS_GAUGES = {
+      'busy' => 'Number of running jobs',
+      'concurrency' => 'Maximum concurrency',
+    }.freeze
+
+    attr_reader :sidekiq_metrics, :gauges
+
+    def initialize
+      @sidekiq_metrics = []
+      @gauges = {}
+    end
+
+    def type
+      'sidekiq_process'
+    end
+
+    def metrics
+      sidekiq_metrics.map do |metric|
+        labels = metric.fetch('labels', {})
+        SIDEKIQ_PROCESS_GAUGES.map do |name, help|
+          if (value = metric[name])
+            gauge = gauges[name] ||= PrometheusExporter::Metric::Gauge.new("sidekiq_process_#{name}", help)
+            gauge.observe(value, labels)
+          end
+        end
+      end
+
+      gauges.values
+    end
+
+    def collect(object)
+      now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      process = object['process']
+
+      process["created_at"] = now
+      sidekiq_metrics.delete_if { |metric| metric['created_at'] + MAX_SIDEKIQ_METRIC_AGE < now }
+      sidekiq_metrics << process
+    end
+  end
+end

data/lib/prometheus_exporter/server/sidekiq_queue_collector.rb

@@ -4,7 +4,7 @@ module PrometheusExporter::Server
     MAX_SIDEKIQ_METRIC_AGE = 60
 
     SIDEKIQ_QUEUE_GAUGES = {
-      'backlog_total' => 'Size of the sidekiq queue.',
+      'backlog' => 'Size of the sidekiq queue.',
       'latency_seconds' => 'Latency of the sidekiq queue.',
     }.freeze
 

data/lib/prometheus_exporter/server/sidekiq_stats_collector.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module PrometheusExporter::Server
+  class SidekiqStatsCollector < TypeCollector
+    MAX_SIDEKIQ_METRIC_AGE = 60
+
+    SIDEKIQ_STATS_GAUGES = {
+      'dead_size' => 'Size of dead the queue',
+      'enqueued' => 'Number of enqueued jobs',
+      'failed' => 'Number of failed jobs',
+      'processed' => 'Total number of processed jobs',
+      'processes_size' => 'Number of processes',
+      'retry_size' => 'Size of the retries queue',
+      'scheduled_size' => 'Size of the scheduled queue',
+      'workers_size' => 'Number of jobs actively being processed',
+    }.freeze
+
+    attr_reader :sidekiq_metrics, :gauges
+
+    def initialize
+      @sidekiq_metrics = []
+      @gauges = {}
+    end
+
+    def type
+      'sidekiq_stats'
+    end
+
+    def metrics
+      sidekiq_metrics.map do |metric|
+        SIDEKIQ_STATS_GAUGES.map do |name, help|
+          if (value = metric['stats'][name])
+            gauge = gauges[name] ||= PrometheusExporter::Metric::Gauge.new("sidekiq_stats_#{name}", help)
+            gauge.observe(value)
+          end
+        end
+      end
+
+      gauges.values
+    end
+
+    def collect(object)
+      sidekiq_metrics << object
+    end
+  end
+end

data/lib/prometheus_exporter/server/unicorn_collector.rb

@@ -6,9 +6,9 @@ class PrometheusExporter::Server::UnicornCollector < PrometheusExporter::Server:
   MAX_UNICORN_METRIC_AGE = 60
 
   UNICORN_GAUGES = {
-    workers_total: 'Number of unicorn workers.',
-    active_workers_total: 'Number of active unicorn workers',
-    request_backlog_total: 'Number of requests waiting to be processed by a unicorn worker.'
+    workers: 'Number of unicorn workers.',
+    active_workers: 'Number of active unicorn workers',
+    request_backlog: 'Number of requests waiting to be processed by a unicorn worker.'
   }.freeze
 
   def initialize

data/lib/prometheus_exporter/server/web_collector.rb

@@ -33,22 +33,22 @@ module PrometheusExporter::Server
           "Total HTTP requests from web app."
         )
 
-        @metrics["http_duration_seconds"] = @http_duration_seconds = PrometheusExporter::Metric::Summary.new(
+        @metrics["http_duration_seconds"] = @http_duration_seconds = PrometheusExporter::Metric::Base.default_aggregation.new(
           "http_duration_seconds",
           "Time spent in HTTP reqs in seconds."
         )
 
-        @metrics["http_redis_duration_seconds"] = @http_redis_duration_seconds = PrometheusExporter::Metric::Summary.new(
+        @metrics["http_redis_duration_seconds"] = @http_redis_duration_seconds = PrometheusExporter::Metric::Base.default_aggregation.new(
           "http_redis_duration_seconds",
           "Time spent in HTTP reqs in Redis, in seconds."
         )
 
-        @metrics["http_sql_duration_seconds"] = @http_sql_duration_seconds = PrometheusExporter::Metric::Summary.new(
+        @metrics["http_sql_duration_seconds"] = @http_sql_duration_seconds = PrometheusExporter::Metric::Base.default_aggregation.new(
           "http_sql_duration_seconds",
           "Time spent in HTTP reqs in SQL in seconds."
         )
 
-        @metrics["http_queue_duration_seconds"] = @http_queue_duration_seconds = PrometheusExporter::Metric::Summary.new(
+        @metrics["http_queue_duration_seconds"] = @http_queue_duration_seconds = PrometheusExporter::Metric::Base.default_aggregation.new(
           "http_queue_duration_seconds",
           "Time spent queueing the request in load balancer in seconds."
         )

data/lib/prometheus_exporter/server/web_server.rb

@@ -29,16 +29,17 @@ module PrometheusExporter::Server
       @bad_metrics_total.observe(0)
 
       @access_log, @logger = nil
+      log_target = opts[:log_target]
 
       if @verbose
         @access_log = [
           [$stderr, WEBrick::AccessLog::COMMON_LOG_FORMAT],
           [$stderr, WEBrick::AccessLog::REFERER_LOG_FORMAT],
         ]
-        @logger = WEBrick::Log.new($stderr)
+        @logger = WEBrick::Log.new(log_target || $stderr)
       else
         @access_log = []
-        @logger = WEBrick::Log.new("/dev/null")
+        @logger = WEBrick::Log.new(log_target || "/dev/null")
       end
 
       @logger.info "Using Basic Authentication via #{@auth}" if @verbose && @auth
@@ -87,10 +88,7 @@ module PrometheusExporter::Server
           @collector.process(block)
         rescue => e
           if @verbose
-            STDERR.puts
-            STDERR.puts e.inspect
-            STDERR.puts e.backtrace
-            STDERR.puts
+            @logger.error "\n\n#{e.inspect}\n#{e.backtrace}\n\n"
           end
           @bad_metrics_total.observe
           res.body = "Bad Metrics #{e}"
@@ -108,7 +106,7 @@ module PrometheusExporter::Server
         begin
           @server.start
         rescue => e
-          STDERR.puts "Failed to start prometheus collector web on port #{@port}: #{e}"
+          @logger.error "Failed to start prometheus collector web on port #{@port}: #{e}"
         end
       end
     end
@@ -125,7 +123,7 @@ module PrometheusExporter::Server
         end
       rescue Timeout::Error
         # we timed out ... bummer
-        STDERR.puts "Generating Prometheus metrics text timed out"
+        @logger.error "Generating Prometheus metrics text timed out"
       end
 
       metrics = []

data/lib/prometheus_exporter/server.rb

@@ -6,6 +6,8 @@ require_relative "server/web_collector"
 require_relative "server/process_collector"
 require_relative "server/sidekiq_collector"
 require_relative "server/sidekiq_queue_collector"
+require_relative "server/sidekiq_process_collector"
+require_relative "server/sidekiq_stats_collector"
 require_relative "server/delayed_job_collector"
 require_relative "server/collector_base"
 require_relative "server/collector"
@@ -16,3 +18,4 @@ require_relative "server/hutch_collector"
 require_relative "server/unicorn_collector"
 require_relative "server/active_record_collector"
 require_relative "server/shoryuken_collector"
+require_relative "server/resque_collector"

data/lib/prometheus_exporter/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PrometheusExporter
-  VERSION = '0.7.0'
+  VERSION = '1.0.1'
 end

data/prometheus_exporter.gemspec

@@ -27,11 +27,11 @@ Gem::Specification.new do |spec|
   spec.add_dependency "webrick"
 
   spec.add_development_dependency "rubocop", ">= 0.69"
-  spec.add_development_dependency "bundler", ">= 2.2.2"
+  spec.add_development_dependency "bundler", ">= 2.1.4"
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "minitest", "~> 5.0"
   spec.add_development_dependency "guard", "~> 2.0"
-  spec.add_development_dependency "mini_racer", "~> 0.3.1"
+  spec.add_development_dependency "mini_racer", "~> 0.5.0"
   spec.add_development_dependency "guard-minitest", "~> 2.0"
   spec.add_development_dependency "oj", "~> 3.0"
   spec.add_development_dependency "rack-test", "~> 0.8.3"
@@ -42,5 +42,5 @@ Gem::Specification.new do |spec|
   if !RUBY_ENGINE == 'jruby'
     spec.add_development_dependency "raindrops", "~> 0.19"
   end
-  spec.required_ruby_version = '>= 2.5.0'
+  spec.required_ruby_version = '>= 2.6.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: prometheus_exporter
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 1.0.1
 platform: ruby
 authors:
 - Sam Saffron
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-12-28 00:00:00.000000000 Z
+date: 2021-12-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: webrick
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.2
+        version: 2.1.4
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.2
+        version: 2.1.4
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -100,14 +100,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.1
+        version: 0.5.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.1
+        version: 0.5.0
 - !ruby/object:Gem::Dependency
   name: guard-minitest
   requirement: !ruby/object:Gem::Requirement
@@ -240,9 +240,12 @@ files:
 - lib/prometheus_exporter/instrumentation/method_profiler.rb
 - lib/prometheus_exporter/instrumentation/process.rb
 - lib/prometheus_exporter/instrumentation/puma.rb
+- lib/prometheus_exporter/instrumentation/resque.rb
 - lib/prometheus_exporter/instrumentation/shoryuken.rb
 - lib/prometheus_exporter/instrumentation/sidekiq.rb
+- lib/prometheus_exporter/instrumentation/sidekiq_process.rb
 - lib/prometheus_exporter/instrumentation/sidekiq_queue.rb
+- lib/prometheus_exporter/instrumentation/sidekiq_stats.rb
 - lib/prometheus_exporter/instrumentation/unicorn.rb
 - lib/prometheus_exporter/metric.rb
 - lib/prometheus_exporter/metric/base.rb
@@ -259,10 +262,13 @@ files:
 - lib/prometheus_exporter/server/hutch_collector.rb
 - lib/prometheus_exporter/server/process_collector.rb
 - lib/prometheus_exporter/server/puma_collector.rb
+- lib/prometheus_exporter/server/resque_collector.rb
 - lib/prometheus_exporter/server/runner.rb
 - lib/prometheus_exporter/server/shoryuken_collector.rb
 - lib/prometheus_exporter/server/sidekiq_collector.rb
+- lib/prometheus_exporter/server/sidekiq_process_collector.rb
 - lib/prometheus_exporter/server/sidekiq_queue_collector.rb
+- lib/prometheus_exporter/server/sidekiq_stats_collector.rb
 - lib/prometheus_exporter/server/type_collector.rb
 - lib/prometheus_exporter/server/unicorn_collector.rb
 - lib/prometheus_exporter/server/web_collector.rb
@@ -282,15 +288,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.5.0
+      version: 2.6.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.2
-signing_key:
+rubygems_version: 3.1.6
+signing_key: 
 specification_version: 4
 summary: Prometheus Exporter
 test_files: []