prometheus_exporter 0.4.9 → 0.5.0

data/lib/prometheus_exporter/metric/gauge.rb

@@ -6,7 +6,7 @@ module PrometheusExporter::Metric
 
     def initialize(name, help)
       super
-      @data = {}
+      reset!
     end
 
     def type
@@ -19,6 +19,14 @@ module PrometheusExporter::Metric
       end.join("\n")
     end
 
+    def reset!
+      @data = {}
+    end
+
+    def to_h
+      @data.dup
+    end
+
     def observe(value, labels = {})
       if value.nil?
         data.delete(labels)

data/lib/prometheus_exporter/metric/histogram.rb

@@ -8,11 +8,25 @@ module PrometheusExporter::Metric
     def initialize(name, help, opts = {})
       super(name, help)
       @buckets = (opts[:buckets] || DEFAULT_BUCKETS).sort.reverse
+      reset!
+    end
+
+    def reset!
       @sums = {}
       @counts = {}
       @observations = {}
     end
 
+    def to_h
+      data = {}
+      @observations.each do |labels, buckets|
+        count = @counts[labels]
+        sum = @sums[labels]
+        data[labels] = { "count" => count, "sum" => sum }
+      end
+      data
+    end
+
     def type
       "histogram"
     end

data/lib/prometheus_exporter/metric/summary.rb

@@ -10,12 +10,26 @@ module PrometheusExporter::Metric
 
     def initialize(name, help, opts = {})
       super(name, help)
+      reset!
+      @quantiles = opts[:quantiles] || DEFAULT_QUANTILES
+    end
+
+    def reset!
       @buffers = [{}, {}]
       @last_rotated = Process.clock_gettime(Process::CLOCK_MONOTONIC)
       @current_buffer = 0
       @counts = {}
       @sums = {}
-      @quantiles = opts[:quantiles] || DEFAULT_QUANTILES
+    end
+
+    def to_h
+      data = {}
+      calculate_all_quantiles.each do |labels, quantiles|
+        count = @counts[labels]
+        sum = @sums[labels]
+        data[labels] = { "count" => count, "sum" => sum }
+      end
+      data
     end
 
     def type

data/lib/prometheus_exporter/metric.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative "metric/base"
 require_relative "metric/counter"
 require_relative "metric/gauge"

data/lib/prometheus_exporter/server/active_record_collector.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module PrometheusExporter::Server
+  class ActiveRecordCollector < TypeCollector
+    MAX_ACTIVERECORD_METRIC_AGE = 60
+    ACTIVE_RECORD_GAUGES = {
+      connections: "Total connections in pool",
+      busy: "Connections in use in pool",
+      dead: "Dead connections in pool",
+      idle: "Idle connections in pool",
+      waiting: "Connection requests waiting",
+      size: "Maximum allowed connection pool size"
+    }
+
+    def initialize
+      @active_record_metrics = []
+    end
+
+    def type
+      "active_record"
+    end
+
+    def metrics
+      return [] if @active_record_metrics.length == 0
+
+      metrics = {}
+
+      @active_record_metrics.map do |m|
+        metric_key = (m["metric_labels"] || {}).merge("pid" => m["pid"])
+
+        ACTIVE_RECORD_GAUGES.map do |k, help|
+          k = k.to_s
+          if v = m[k]
+            g = metrics[k] ||= PrometheusExporter::Metric::Gauge.new("active_record_connection_pool_#{k}", help)
+            g.observe(v, metric_key)
+          end
+        end
+      end
+
+      metrics.values
+    end
+
+    def collect(obj)
+      now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+
+      obj["created_at"] = now
+
+      @active_record_metrics.delete_if do |current|
+        (obj["pid"] == current["pid"] && obj["hostname"] == current["hostname"]) ||
+          (current["created_at"] + MAX_ACTIVERECORD_METRIC_AGE < now)
+      end
+
+      @active_record_metrics << obj
+    end
+  end
+end

data/lib/prometheus_exporter/server/collector.rb

@@ -17,6 +17,8 @@ module PrometheusExporter::Server
       register_collector(PumaCollector.new)
       register_collector(HutchCollector.new)
       register_collector(UnicornCollector.new)
+      register_collector(ActiveRecordCollector.new)
+      register_collector(ShoryukenCollector.new)
     end
 
     def register_collector(collector)
@@ -72,6 +74,7 @@ module PrometheusExporter::Server
     def register_metric_unsafe(obj)
       name = obj["name"]
       help = obj["help"]
+      opts = symbolize_keys(obj["opts"] || {})
 
       metric =
         case obj["type"]
@@ -80,9 +83,9 @@ module PrometheusExporter::Server
         when "counter"
           PrometheusExporter::Metric::Counter.new(name, help)
         when "summary"
-          PrometheusExporter::Metric::Summary.new(name, help)
+          PrometheusExporter::Metric::Summary.new(name, help, opts)
         when "histogram"
-          PrometheusExporter::Metric::Histogram.new(name, help)
+          PrometheusExporter::Metric::Histogram.new(name, help, opts)
         end
 
       if metric
@@ -91,5 +94,9 @@ module PrometheusExporter::Server
         STDERR.puts "failed to register metric #{obj}"
       end
     end
+
+    def symbolize_keys(hash)
+      hash.inject({}) { |memo, k| memo[k.first.to_sym] = k.last; memo }
+    end
   end
 end

data/lib/prometheus_exporter/server/collector_base.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module PrometheusExporter::Server
 
   # minimal interface to implement a customer collector

data/lib/prometheus_exporter/server/delayed_job_collector.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module PrometheusExporter::Server
   class DelayedJobCollector < TypeCollector
 
@@ -19,12 +21,15 @@ module PrometheusExporter::Server
       @delayed_job_duration_seconds_summary.observe(obj["duration"], status: "success") if obj["success"]
       @delayed_job_duration_seconds_summary.observe(obj["duration"], status: "failed")  if !obj["success"]
       @delayed_job_attempts_summary.observe(obj["attempts"]) if obj["success"]
+      @delayed_jobs_enqueued.observe(obj["enqueued"])
+      @delayed_jobs_pending.observe(obj["pending"])
     end
 
     def metrics
       if @delayed_jobs_total
         [@delayed_job_duration_seconds, @delayed_jobs_total, @delayed_failed_jobs_total,
-         @delayed_jobs_max_attempts_reached_total, @delayed_job_duration_seconds_summary, @delayed_job_attempts_summary]
+         @delayed_jobs_max_attempts_reached_total, @delayed_job_duration_seconds_summary, @delayed_job_attempts_summary,
+         @delayed_jobs_enqueued, @delayed_jobs_pending]
       else
         []
       end
@@ -43,6 +48,14 @@ module PrometheusExporter::Server
         PrometheusExporter::Metric::Counter.new(
           "delayed_jobs_total", "Total number of delayed jobs executed.")
 
+        @delayed_jobs_enqueued =
+        PrometheusExporter::Metric::Gauge.new(
+          "delayed_jobs_enqueued", "Number of enqueued delayed jobs.")
+
+        @delayed_jobs_pending =
+        PrometheusExporter::Metric::Gauge.new(
+          "delayed_jobs_pending", "Number of pending delayed jobs.")
+
         @delayed_failed_jobs_total =
         PrometheusExporter::Metric::Counter.new(
           "delayed_failed_jobs_total", "Total number failed delayed jobs executed.")

data/lib/prometheus_exporter/server/hutch_collector.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module PrometheusExporter::Server
   class HutchCollector < TypeCollector
 

data/lib/prometheus_exporter/server/process_collector.rb

@@ -35,6 +35,7 @@ module PrometheusExporter::Server
 
       @process_metrics.map do |m|
         metric_key = m["metric_labels"].merge("pid" => m["pid"])
+        metric_key.merge!(m["custom_labels"] || {})
 
         PROCESS_GAUGES.map do |k, help|
           k = k.to_s

data/lib/prometheus_exporter/server/puma_collector.rb

@@ -1,5 +1,8 @@
+# frozen_string_literal: true
+
 module PrometheusExporter::Server
   class PumaCollector < TypeCollector
+    MAX_PUMA_METRIC_AGE = 30
     PUMA_GAUGES = {
       workers_total: "Number of puma workers.",
       booted_workers_total: "Number of puma workers booted.",
@@ -26,7 +29,10 @@ module PrometheusExporter::Server
       @puma_metrics.map do |m|
         labels = {}
         if m["phase"]
-          labels.merge(phase: m["phase"])
+          labels.merge!(phase: m["phase"])
+        end
+        if m["custom_labels"]
+          labels.merge!(m["custom_labels"])
         end
 
         PUMA_GAUGES.map do |k, help|
@@ -42,6 +48,10 @@ module PrometheusExporter::Server
     end
 
     def collect(obj)
+      now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+
+      obj["created_at"] = now
+      @puma_metrics.delete_if { |m| m["created_at"] + MAX_PUMA_METRIC_AGE < now }
       @puma_metrics << obj
     end
   end

data/lib/prometheus_exporter/server/runner.rb

@@ -1,10 +1,11 @@
 # frozen_string_literal: true
+
 require 'prometheus_exporter/client'
 require_relative '../instrumentation/unicorn'
 
 module PrometheusExporter::Server
-  class RunnerException < StandardError; end;
-  class WrongInheritance < RunnerException; end;
+  class RunnerException < StandardError; end
+  class WrongInheritance < RunnerException; end
 
   class Runner
     def initialize(options = {})
@@ -31,69 +32,46 @@ module PrometheusExporter::Server
         )
       end
 
-      server = server_class.new port: port, collector: collector, timeout: timeout, verbose: verbose
+      server = server_class.new port: port, bind: bind, collector: collector, timeout: timeout, verbose: verbose
       server.start
     end
 
-    def prefix=(prefix)
-      @prefix = prefix
-    end
+    attr_accessor :unicorn_listen_address, :unicorn_pid_file
+    attr_writer :prefix, :port, :bind, :collector_class, :type_collectors, :timeout, :verbose, :server_class
 
     def prefix
       @prefix || PrometheusExporter::DEFAULT_PREFIX
     end
 
-    def port=(port)
-      @port = port
-    end
-
     def port
       @port || PrometheusExporter::DEFAULT_PORT
     end
 
-    def collector_class=(collector_class)
-      @collector_class = collector_class
+    def bind
+      @bind || PrometheusExporter::DEFAULT_BIND_ADDRESS
     end
 
     def collector_class
       @collector_class || PrometheusExporter::Server::Collector
     end
 
-    def type_collectors=(type_collectors)
-      @type_collectors = type_collectors
-    end
-
     def type_collectors
       @type_collectors || []
     end
 
-    def timeout=(timeout)
-      @timeout = timeout
-    end
-
     def timeout
       @timeout || PrometheusExporter::DEFAULT_TIMEOUT
     end
 
-    def verbose=(verbose)
-      @verbose = verbose
-    end
-
     def verbose
       return @verbose if defined? @verbose
       false
     end
 
-    def server_class=(server_class)
-      @server_class = server_class
-    end
-
     def server_class
       @server_class || PrometheusExporter::Server::WebServer
     end
 
-    attr_accessor :unicorn_listen_address, :unicorn_pid_file
-
     def collector
       @_collector ||= collector_class.new
     end

data/lib/prometheus_exporter/server/shoryuken_collector.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module PrometheusExporter::Server
+  class ShoryukenCollector < TypeCollector
+
+    def type
+      "shoryuken"
+    end
+
+    def collect(obj)
+      default_labels = { job_name: obj['name'] , queue_name: obj['queue'] }
+      custom_labels = obj['custom_labels']
+      labels = custom_labels.nil? ? default_labels : default_labels.merge(custom_labels)
+
+      ensure_shoryuken_metrics
+      @shoryuken_job_duration_seconds.observe(obj["duration"], labels)
+      @shoryuken_jobs_total.observe(1, labels)
+      @shoryuken_restarted_jobs_total.observe(1, labels) if obj["shutdown"]
+      @shoryuken_failed_jobs_total.observe(1, labels) if !obj["success"] && !obj["shutdown"]
+    end
+
+    def metrics
+      if @shoryuken_jobs_total
+        [
+            @shoryuken_job_duration_seconds,
+            @shoryuken_jobs_total,
+            @shoryuken_restarted_jobs_total,
+            @shoryuken_failed_jobs_total,
+        ]
+      else
+        []
+      end
+    end
+
+    protected
+
+    def ensure_shoryuken_metrics
+      if !@shoryuken_jobs_total
+
+        @shoryuken_job_duration_seconds =
+            PrometheusExporter::Metric::Counter.new(
+                "shoryuken_job_duration_seconds", "Total time spent in shoryuken jobs.")
+
+        @shoryuken_jobs_total =
+            PrometheusExporter::Metric::Counter.new(
+                "shoryuken_jobs_total", "Total number of shoryuken jobs executed.")
+
+        @shoryuken_restarted_jobs_total =
+            PrometheusExporter::Metric::Counter.new(
+                "shoryuken_restarted_jobs_total", "Total number of shoryuken jobs that we restarted because of a shoryuken shutdown.")
+
+        @shoryuken_failed_jobs_total =
+            PrometheusExporter::Metric::Counter.new(
+                "shoryuken_failed_jobs_total", "Total number of failed shoryuken jobs.")
+
+      end
+    end
+  end
+end

data/lib/prometheus_exporter/server/sidekiq_collector.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module PrometheusExporter::Server
   class SidekiqCollector < TypeCollector
 

data/lib/prometheus_exporter/server/type_collector.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module PrometheusExporter::Server
   class TypeCollector
     def type

data/lib/prometheus_exporter/server/unicorn_collector.rb

@@ -3,6 +3,8 @@
 # custom type collector for prometheus_exporter for handling the metrics sent from
 # PrometheusExporter::Instrumentation::Unicorn
 class PrometheusExporter::Server::UnicornCollector < PrometheusExporter::Server::TypeCollector
+  MAX_UNICORN_METRIC_AGE = 60
+
   UNICORN_GAUGES = {
     workers_total: 'Number of unicorn workers.',
     active_workers_total: 'Number of active unicorn workers',
@@ -23,11 +25,13 @@ class PrometheusExporter::Server::UnicornCollector < PrometheusExporter::Server:
     metrics = {}
 
     @unicorn_metrics.map do |m|
+      labels = m["custom_labels"] || {}
+
       UNICORN_GAUGES.map do |k, help|
         k = k.to_s
         if (v = m[k])
           g = metrics[k] ||= PrometheusExporter::Metric::Gauge.new("unicorn_#{k}", help)
-          g.observe(v)
+          g.observe(v, labels)
         end
       end
     end
@@ -36,6 +40,9 @@ class PrometheusExporter::Server::UnicornCollector < PrometheusExporter::Server:
   end
 
   def collect(obj)
+    now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+    obj["created_at"] = now
+    @unicorn_metrics.delete_if { |m| m['created_at'] + MAX_UNICORN_METRIC_AGE < now }
     @unicorn_metrics << obj
   end
 end

data/lib/prometheus_exporter/server/web_server.rb

@@ -9,7 +9,7 @@ module PrometheusExporter::Server
   class WebServer
     attr_reader :collector
 
-    def initialize(port: , collector: nil, timeout: PrometheusExporter::DEFAULT_TIMEOUT, verbose: false)
+    def initialize(port: , bind: nil, collector: nil, timeout: PrometheusExporter::DEFAULT_TIMEOUT, verbose: false)
 
       @verbose = verbose
 
@@ -38,6 +38,7 @@ module PrometheusExporter::Server
 
       @server = WEBrick::HTTPServer.new(
         Port: port,
+        BindAddress: bind,
         Logger: logger,
         AccessLog: access_log,
       )

data/lib/prometheus_exporter/server.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative "metric"
 require_relative "server/type_collector"
 require_relative "server/web_collector"
@@ -11,3 +13,5 @@ require_relative "server/runner"
 require_relative "server/puma_collector"
 require_relative "server/hutch_collector"
 require_relative "server/unicorn_collector"
+require_relative "server/active_record_collector"
+require_relative "server/shoryuken_collector"

data/lib/prometheus_exporter/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module PrometheusExporter
-  VERSION = "0.4.9"
+  VERSION = '0.5.0'
 end

data/lib/prometheus_exporter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative "prometheus_exporter/version"
 require "json"
 require "thread"
@@ -5,6 +7,7 @@ require "thread"
 module PrometheusExporter
   # per: https://github.com/prometheus/prometheus/wiki/Default-port-allocations
   DEFAULT_PORT = 9394
+  DEFAULT_BIND_ADDRESS = 'localhost'
   DEFAULT_PREFIX = 'ruby_'
   DEFAULT_TIMEOUT = 2
 
@@ -17,6 +20,17 @@ module PrometheusExporter
     end
   end
 
+  def self.hostname
+    @hostname ||=
+      begin
+        require 'socket'
+        Socket.gethostname
+      rescue => e
+        STDERR.puts "Unable to lookup hostname #{e}"
+        "unknown-host"
+      end
+  end
+
   def self.detect_json_serializer(preferred)
     if preferred.nil?
       preferred = :oj if has_oj?

data/prometheus_exporter.gemspec

@@ -5,24 +5,27 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require "prometheus_exporter/version"
 
 Gem::Specification.new do |spec|
-  spec.name          = "prometheus_exporter"
-  spec.version       = PrometheusExporter::VERSION
-  spec.authors       = ["Sam Saffron"]
-  spec.email         = ["sam.saffron@gmail.com"]
+  spec.name                 = "prometheus_exporter"
+  spec.version              = PrometheusExporter::VERSION
+  spec.authors              = ["Sam Saffron"]
+  spec.email                = ["sam.saffron@gmail.com"]
 
-  spec.summary       = %q{Prometheus Exporter}
-  spec.description   = %q{Prometheus metric collector and exporter for Ruby}
-  spec.homepage      = "https://github.com/discourse/prometheus_exporter"
-  spec.license       = "MIT"
+  spec.summary              = %q{Prometheus Exporter}
+  spec.description          = %q{Prometheus metric collector and exporter for Ruby}
+  spec.homepage             = "https://github.com/discourse/prometheus_exporter"
+  spec.license              = "MIT"
 
-  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+  spec.post_install_message = "prometheus_exporter will only bind to localhost by default as of v0.5"
+
+  spec.files = `git ls-files -z`.split("\x0").reject do |f|
     f.match(%r{^(test|spec|features|bin)/})
   end
-  spec.bindir        = "bin"
-  spec.executables   = ["prometheus_exporter"]
-  spec.require_paths = ["lib"]
+  spec.bindir               = "bin"
+  spec.executables          = ["prometheus_exporter"]
+  spec.require_paths        = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.16"
+  spec.add_development_dependency "rubocop", ">= 0.69"
+  spec.add_development_dependency "bundler", "> 1.16"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "minitest", "~> 5.0"
   spec.add_development_dependency "guard", "~> 2.0"
@@ -31,6 +34,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "oj", "~> 3.0"
   spec.add_development_dependency "rack-test", "~> 0.8.3"
   spec.add_development_dependency "minitest-stub-const", "~> 0.6"
+  spec.add_development_dependency 'rubocop-discourse', '~> 1.0'
   if !RUBY_ENGINE == 'jruby'
     spec.add_development_dependency "raindrops", "~> 0.19"
   end