prometheus_exporter 0.8.0 → 2.0.0

data/lib/prometheus_exporter/server/puma_collector.rb

@@ -4,13 +4,13 @@ module PrometheusExporter::Server
   class PumaCollector < TypeCollector
     MAX_PUMA_METRIC_AGE = 30
     PUMA_GAUGES = {
-      workers_total: "Number of puma workers.",
-      booted_workers_total: "Number of puma workers booted.",
-      old_workers_total: "Number of old puma workers.",
-      running_threads_total: "Number of puma threads currently running.",
-      request_backlog_total: "Number of requests waiting to be processed by a puma thread.",
-      thread_pool_capacity_total: "Number of puma threads available at current scale.",
-      max_threads_total: "Number of puma threads at available at max scale.",
+      workers: "Number of puma workers.",
+      booted_workers: "Number of puma workers booted.",
+      old_workers: "Number of old puma workers.",
+      running_threads: "Number of puma threads currently running.",
+      request_backlog: "Number of requests waiting to be processed by a puma thread.",
+      thread_pool_capacity: "Number of puma threads available at current scale.",
+      max_threads: "Number of puma threads at available at max scale.",
     }
 
     def initialize

data/lib/prometheus_exporter/server/resque_collector.rb

@@ -4,12 +4,12 @@ module PrometheusExporter::Server
   class ResqueCollector < TypeCollector
     MAX_RESQUE_METRIC_AGE = 30
     RESQUE_GAUGES = {
-      processed_jobs_total: "Total number of processed Resque jobs.",
-      failed_jobs_total: "Total number of failed Resque jobs.",
-      pending_jobs_total: "Total number of pending Resque jobs.",
-      queues_total: "Total number of Resque queues.",
-      workers_total: "Total number of Resque workers running.",
-      working_total: "Total number of Resque workers working."
+      processed_jobs: "Total number of processed Resque jobs.",
+      failed_jobs: "Total number of failed Resque jobs.",
+      pending_jobs: "Total number of pending Resque jobs.",
+      queues: "Total number of Resque queues.",
+      workers: "Total number of Resque workers running.",
+      working: "Total number of Resque workers working."
     }
 
     def initialize

data/lib/prometheus_exporter/server/runner.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'prometheus_exporter/client'
+require_relative '../client'
 require_relative '../instrumentation/unicorn'
 
 module PrometheusExporter::Server
@@ -17,6 +17,7 @@ module PrometheusExporter::Server
       @prefix = nil
       @auth = nil
       @realm = nil
+      @histogram = nil
 
       options.each do |k, v|
         send("#{k}=", v) if self.class.method_defined?("#{k}=")
@@ -27,6 +28,10 @@ module PrometheusExporter::Server
       PrometheusExporter::Metric::Base.default_prefix = prefix
       PrometheusExporter::Metric::Base.default_labels = label
 
+      if histogram
+        PrometheusExporter::Metric::Base.default_aggregation = PrometheusExporter::Metric::Histogram
+      end
+
       register_type_collectors
 
       unless collector.is_a?(PrometheusExporter::Server::CollectorBase)
@@ -47,7 +52,7 @@ module PrometheusExporter::Server
     end
 
     attr_accessor :unicorn_listen_address, :unicorn_pid_file
-    attr_writer :prefix, :port, :bind, :collector_class, :type_collectors, :timeout, :verbose, :server_class, :label, :auth, :realm
+    attr_writer :prefix, :port, :bind, :collector_class, :type_collectors, :timeout, :verbose, :server_class, :label, :auth, :realm, :histogram
 
     def auth
       @auth || nil
@@ -98,6 +103,10 @@ module PrometheusExporter::Server
       @label ||= PrometheusExporter::DEFAULT_LABEL
     end
 
+    def histogram
+      @histogram || false
+    end
+
     private
 
     def register_type_collectors

data/lib/prometheus_exporter/server/sidekiq_collector.rb

@@ -52,7 +52,7 @@ module PrometheusExporter::Server
       if !@sidekiq_jobs_total
 
         @sidekiq_job_duration_seconds =
-        PrometheusExporter::Metric::Summary.new(
+        PrometheusExporter::Metric::Base.default_aggregation.new(
           "sidekiq_job_duration_seconds", "Total time spent in sidekiq jobs.")
 
         @sidekiq_jobs_total =

data/lib/prometheus_exporter/server/sidekiq_process_collector.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module PrometheusExporter::Server
+  class SidekiqProcessCollector < PrometheusExporter::Server::TypeCollector
+    MAX_SIDEKIQ_METRIC_AGE = 60
+
+    SIDEKIQ_PROCESS_GAUGES = {
+      'busy' => 'Number of running jobs',
+      'concurrency' => 'Maximum concurrency',
+    }.freeze
+
+    attr_reader :sidekiq_metrics, :gauges
+
+    def initialize
+      @sidekiq_metrics = []
+      @gauges = {}
+    end
+
+    def type
+      'sidekiq_process'
+    end
+
+    def metrics
+      sidekiq_metrics.map do |metric|
+        labels = metric.fetch('labels', {})
+        SIDEKIQ_PROCESS_GAUGES.map do |name, help|
+          if (value = metric[name])
+            gauge = gauges[name] ||= PrometheusExporter::Metric::Gauge.new("sidekiq_process_#{name}", help)
+            gauge.observe(value, labels)
+          end
+        end
+      end
+
+      gauges.values
+    end
+
+    def collect(object)
+      now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      process = object['process']
+
+      process["created_at"] = now
+      sidekiq_metrics.delete_if { |metric| metric['created_at'] + MAX_SIDEKIQ_METRIC_AGE < now }
+      sidekiq_metrics << process
+    end
+  end
+end

data/lib/prometheus_exporter/server/sidekiq_queue_collector.rb

@@ -4,7 +4,7 @@ module PrometheusExporter::Server
     MAX_SIDEKIQ_METRIC_AGE = 60
 
     SIDEKIQ_QUEUE_GAUGES = {
-      'backlog_total' => 'Size of the sidekiq queue.',
+      'backlog' => 'Size of the sidekiq queue.',
       'latency_seconds' => 'Latency of the sidekiq queue.',
     }.freeze
 

data/lib/prometheus_exporter/server/sidekiq_stats_collector.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module PrometheusExporter::Server
+  class SidekiqStatsCollector < TypeCollector
+    MAX_SIDEKIQ_METRIC_AGE = 60
+
+    SIDEKIQ_STATS_GAUGES = {
+      'dead_size' => 'Size of dead the queue',
+      'enqueued' => 'Number of enqueued jobs',
+      'failed' => 'Number of failed jobs',
+      'processed' => 'Total number of processed jobs',
+      'processes_size' => 'Number of processes',
+      'retry_size' => 'Size of the retries queue',
+      'scheduled_size' => 'Size of the scheduled queue',
+      'workers_size' => 'Number of jobs actively being processed',
+    }.freeze
+
+    attr_reader :sidekiq_metrics, :gauges
+
+    def initialize
+      @sidekiq_metrics = []
+      @gauges = {}
+    end
+
+    def type
+      'sidekiq_stats'
+    end
+
+    def metrics
+      sidekiq_metrics.map do |metric|
+        SIDEKIQ_STATS_GAUGES.map do |name, help|
+          if (value = metric['stats'][name])
+            gauge = gauges[name] ||= PrometheusExporter::Metric::Gauge.new("sidekiq_stats_#{name}", help)
+            gauge.observe(value)
+          end
+        end
+      end
+
+      gauges.values
+    end
+
+    def collect(object)
+      sidekiq_metrics << object
+    end
+  end
+end

data/lib/prometheus_exporter/server/unicorn_collector.rb

@@ -6,9 +6,9 @@ class PrometheusExporter::Server::UnicornCollector < PrometheusExporter::Server:
   MAX_UNICORN_METRIC_AGE = 60
 
   UNICORN_GAUGES = {
-    workers_total: 'Number of unicorn workers.',
-    active_workers_total: 'Number of active unicorn workers',
-    request_backlog_total: 'Number of requests waiting to be processed by a unicorn worker.'
+    workers: 'Number of unicorn workers.',
+    active_workers: 'Number of active unicorn workers',
+    request_backlog: 'Number of requests waiting to be processed by a unicorn worker.'
   }.freeze
 
   def initialize

data/lib/prometheus_exporter/server/web_collector.rb

@@ -5,10 +5,10 @@ module PrometheusExporter::Server
     def initialize
       @metrics = {}
       @http_requests_total = nil
-      @http_duration_seconds = nil
-      @http_redis_duration_seconds = nil
-      @http_sql_duration_seconds = nil
-      @http_queue_duration_seconds = nil
+      @http_request_duration_seconds = nil
+      @http_request_redis_duration_seconds = nil
+      @http_request_sql_duration_seconds = nil
+      @http_request_queue_duration_seconds = nil
     end
 
     def type
@@ -33,23 +33,23 @@ module PrometheusExporter::Server
           "Total HTTP requests from web app."
         )
 
-        @metrics["http_duration_seconds"] = @http_duration_seconds = PrometheusExporter::Metric::Summary.new(
-          "http_duration_seconds",
+        @metrics["http_request_duration_seconds"] = @http_request_duration_seconds = PrometheusExporter::Metric::Base.default_aggregation.new(
+          "http_request_duration_seconds",
           "Time spent in HTTP reqs in seconds."
         )
 
-        @metrics["http_redis_duration_seconds"] = @http_redis_duration_seconds = PrometheusExporter::Metric::Summary.new(
-          "http_redis_duration_seconds",
+        @metrics["http_request_redis_duration_seconds"] = @http_request_redis_duration_seconds = PrometheusExporter::Metric::Base.default_aggregation.new(
+          "http_request_redis_duration_seconds",
           "Time spent in HTTP reqs in Redis, in seconds."
         )
 
-        @metrics["http_sql_duration_seconds"] = @http_sql_duration_seconds = PrometheusExporter::Metric::Summary.new(
-          "http_sql_duration_seconds",
+        @metrics["http_request_sql_duration_seconds"] = @http_request_sql_duration_seconds = PrometheusExporter::Metric::Base.default_aggregation.new(
+          "http_request_sql_duration_seconds",
           "Time spent in HTTP reqs in SQL in seconds."
         )
 
-        @metrics["http_queue_duration_seconds"] = @http_queue_duration_seconds = PrometheusExporter::Metric::Summary.new(
-          "http_queue_duration_seconds",
+        @metrics["http_request_queue_duration_seconds"] = @http_request_queue_duration_seconds = PrometheusExporter::Metric::Base.default_aggregation.new(
+          "http_request_queue_duration_seconds",
           "Time spent queueing the request in load balancer in seconds."
         )
       end
@@ -60,19 +60,19 @@ module PrometheusExporter::Server
       custom_labels = obj['custom_labels']
       labels = custom_labels.nil? ? default_labels : default_labels.merge(custom_labels)
 
-      @http_requests_total.observe(1, labels)
+      @http_requests_total.observe(1, labels.merge(status: obj["status"]))
 
       if timings = obj["timings"]
-        @http_duration_seconds.observe(timings["total_duration"], labels)
+        @http_request_duration_seconds.observe(timings["total_duration"], labels)
         if redis = timings["redis"]
-          @http_redis_duration_seconds.observe(redis["duration"], labels)
+          @http_request_redis_duration_seconds.observe(redis["duration"], labels)
         end
         if sql = timings["sql"]
-          @http_sql_duration_seconds.observe(sql["duration"], labels)
+          @http_request_sql_duration_seconds.observe(sql["duration"], labels)
         end
       end
       if queue_time = obj["queue_time"]
-        @http_queue_duration_seconds.observe(queue_time, labels)
+        @http_request_queue_duration_seconds.observe(queue_time, labels)
       end
     end
   end

data/lib/prometheus_exporter/server/web_server.rb

@@ -29,16 +29,17 @@ module PrometheusExporter::Server
       @bad_metrics_total.observe(0)
 
       @access_log, @logger = nil
+      log_target = opts[:log_target]
 
       if @verbose
         @access_log = [
           [$stderr, WEBrick::AccessLog::COMMON_LOG_FORMAT],
           [$stderr, WEBrick::AccessLog::REFERER_LOG_FORMAT],
         ]
-        @logger = WEBrick::Log.new($stderr)
+        @logger = WEBrick::Log.new(log_target || $stderr)
       else
         @access_log = []
-        @logger = WEBrick::Log.new("/dev/null")
+        @logger = WEBrick::Log.new(log_target || "/dev/null")
       end
 
       @logger.info "Using Basic Authentication via #{@auth}" if @verbose && @auth
@@ -87,10 +88,7 @@ module PrometheusExporter::Server
           @collector.process(block)
         rescue => e
           if @verbose
-            STDERR.puts
-            STDERR.puts e.inspect
-            STDERR.puts e.backtrace
-            STDERR.puts
+            @logger.error "\n\n#{e.inspect}\n#{e.backtrace}\n\n"
           end
           @bad_metrics_total.observe
           res.body = "Bad Metrics #{e}"
@@ -108,7 +106,7 @@ module PrometheusExporter::Server
         begin
           @server.start
         rescue => e
-          STDERR.puts "Failed to start prometheus collector web on port #{@port}: #{e}"
+          @logger.error "Failed to start prometheus collector web on port #{@port}: #{e}"
         end
       end
     end
@@ -125,7 +123,7 @@ module PrometheusExporter::Server
         end
       rescue Timeout::Error
         # we timed out ... bummer
-        STDERR.puts "Generating Prometheus metrics text timed out"
+        @logger.error "Generating Prometheus metrics text timed out"
       end
 
       metrics = []

data/lib/prometheus_exporter/server.rb

@@ -6,6 +6,8 @@ require_relative "server/web_collector"
 require_relative "server/process_collector"
 require_relative "server/sidekiq_collector"
 require_relative "server/sidekiq_queue_collector"
+require_relative "server/sidekiq_process_collector"
+require_relative "server/sidekiq_stats_collector"
 require_relative "server/delayed_job_collector"
 require_relative "server/collector_base"
 require_relative "server/collector"

data/lib/prometheus_exporter/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PrometheusExporter
-  VERSION = '0.8.0'
+  VERSION = '2.0.0'
 end

data/prometheus_exporter.gemspec

@@ -31,7 +31,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "minitest", "~> 5.0"
   spec.add_development_dependency "guard", "~> 2.0"
-  spec.add_development_dependency "mini_racer", "~> 0.3.1"
+  spec.add_development_dependency "mini_racer", "~> 0.5.0"
   spec.add_development_dependency "guard-minitest", "~> 2.0"
   spec.add_development_dependency "oj", "~> 3.0"
   spec.add_development_dependency "rack-test", "~> 0.8.3"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: prometheus_exporter
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 2.0.0
 platform: ruby
 authors:
 - Sam Saffron
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-07-05 00:00:00.000000000 Z
+date: 2022-02-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: webrick
@@ -100,14 +100,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.1
+        version: 0.5.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.1
+        version: 0.5.0
 - !ruby/object:Gem::Dependency
   name: guard-minitest
   requirement: !ruby/object:Gem::Requirement
@@ -243,7 +243,9 @@ files:
 - lib/prometheus_exporter/instrumentation/resque.rb
 - lib/prometheus_exporter/instrumentation/shoryuken.rb
 - lib/prometheus_exporter/instrumentation/sidekiq.rb
+- lib/prometheus_exporter/instrumentation/sidekiq_process.rb
 - lib/prometheus_exporter/instrumentation/sidekiq_queue.rb
+- lib/prometheus_exporter/instrumentation/sidekiq_stats.rb
 - lib/prometheus_exporter/instrumentation/unicorn.rb
 - lib/prometheus_exporter/metric.rb
 - lib/prometheus_exporter/metric/base.rb
@@ -264,7 +266,9 @@ files:
 - lib/prometheus_exporter/server/runner.rb
 - lib/prometheus_exporter/server/shoryuken_collector.rb
 - lib/prometheus_exporter/server/sidekiq_collector.rb
+- lib/prometheus_exporter/server/sidekiq_process_collector.rb
 - lib/prometheus_exporter/server/sidekiq_queue_collector.rb
+- lib/prometheus_exporter/server/sidekiq_stats_collector.rb
 - lib/prometheus_exporter/server/type_collector.rb
 - lib/prometheus_exporter/server/unicorn_collector.rb
 - lib/prometheus_exporter/server/web_collector.rb
@@ -292,7 +296,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.1.6
-signing_key:
+signing_key: 
 specification_version: 4
 summary: Prometheus Exporter
 test_files: []