prometheus_exporter 0.4.14 → 0.4.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c0851cbef41df3faf274a45e60e176b3a34906209a82e280624a58fa72ad1929
-  data.tar.gz: d6abfafdd5178e9801df4b09bef7a0597f77549f99036423e3a4c870d6b47db2
+  metadata.gz: 86155ae93296d42918b6419efbdae95665e5149a89083894dd9ba5daaf9f84bf
+  data.tar.gz: 49bee52b66976743b17794f7515cc57c7c6979338d479a66ec49ae0ff30d9f7f
 SHA512:
-  metadata.gz: 7e538b3d2e02c1d20e5a8b4cd30a11c8bbb6f3ffb6d3630384b4e1047ba69fdde331352cbe34cffed58927051a55eac25ab2e012b79700b80ec9052b8696cc67
-  data.tar.gz: d3e52bfbeeaeb14c5044c5a7301370993f900a471ed0c7a2b7c1b58b3881a51f3e144e896e4d6ab011e66dabb06af6cf4cde5de8fade2091f2b557efdc8cca5d
+  metadata.gz: c2ebf589c432b5c64339d15c4c25b99d4cca49228c466ba9a334c4af12b2ac7c7922a8607c9c90ea38cf1afd9a2f2efa5dde087ab4aed8f124e4f2577510ee11
+  data.tar.gz: e1a2f8f93f59b62296ce1a2448f70c3decd8f970787ee5619593080da6e181c02c432e82e52ac9de11f626309eb9aa513e6070c56349ef84fcf3a84d4b44ea45

data/CHANGELOG

@@ -1,3 +1,10 @@
+0.4.15 - 04-11-2019
+
+- FEATURE: Improve delayed job collector, add pending counts
+- FEATURE: New ActiveRecord collector (documented in readme)
+- FEATURE: Allow passing in histogram and summary options
+- FEATURE: Allow custom labels for unicorn collector
+
 0.4.14 - 10-09-2019
 
 - FEATURE: allow finding metrics by name RemoteMetric #find_registered_metric

data/README.md

@@ -13,6 +13,7 @@ To learn more see [Instrumenting Rails with Prometheus](https://samsaffron.com/a
   * [Rails integration](#rails-integration)
     * [Per-process stats](#per-process-stats)
     * [Sidekiq metrics](#sidekiq-metrics)
+    * [ActiveRecord Connection Pool Metrics](#activerecord-connection-pool-metrics)
     * [Delayed Job plugin](#delayed-job-plugin)
     * [Hutch metrics](#hutch-message-processing-tracer)
   * [Puma metrics](#puma-metrics)
@@ -148,6 +149,17 @@ awesome 10
 
 ```
 
+Custom quantiles for summaries and buckets for histograms can also be passed in.
+
+```ruby
+require 'prometheus_exporter/client'
+
+client = PrometheusExporter::Client.default
+histogram = client.register(:histogram, "api_time", "time to call api", buckets: [0.1, 0.5, 1])
+
+histogram.observe(0.2, api: 'twitter')
+```
+
 ### Rails integration
 
 You can easily integrate into any Rack application.
@@ -175,6 +187,60 @@ Ensure you run the exporter in a monitored background process:
 $ bundle exec prometheus_exporter
 ```
 
+#### Activerecord Connection Pool Metrics
+
+This collects activerecord connection pool metrics.
+
+It supports injection of custom labels and the connection config options (`username`, `database`, `host`, `port`) as labels.
+
+For Puma single mode 
+```ruby
+#in puma.rb
+require 'prometheus_exporter/instrumentation'
+PrometheusExporter::Instrumentation::ActiveRecord.start(
+  custom_labels: { type: "puma_single_mode" }, #optional params
+  config_labels: [:database, :host] #optional params
+)
+```
+
+For Puma cluster mode
+
+```ruby
+# in puma.rb
+on_worker_boot do
+  require 'prometheus_exporter/instrumentation'
+  PrometheusExporter::Instrumentation::ActiveRecord.start(
+    custom_labels: { type: "puma_worker" }, #optional params
+    config_labels: [:database, :host] #optional params
+  )
+end
+```
+
+For Unicorn / Passenger
+
+```ruby
+after_fork do 
+  require 'prometheus_exporter/instrumentation'
+  PrometheusExporter::Instrumentation::ActiveRecord.start(
+    custom_labels: { type: "unicorn_worker" }, #optional params
+    config_labels: [:database, :host] #optional params
+  )
+end
+```
+
+For Sidekiq
+```ruby
+Sidekiq.configure_server do |config|
+  config.on :startup do
+    require 'prometheus_exporter/instrumentation'
+    PrometheusExporter::Instrumentation::ActiveRecord.start(
+      custom_labels: { type: "sidekiq" }, #optional params
+      config_labels: [:database, :host] #optional params
+    )
+  end
+end
+```
+
 #### Per-process stats
 
 You may also be interested in per-process stats. This collects memory and GC stats:

data/lib/prometheus_exporter/client.rb

@@ -4,16 +4,16 @@ require 'socket'
 require 'thread'
 
 module PrometheusExporter
-
   class Client
     class RemoteMetric
       attr_reader :name, :type, :help
 
-      def initialize(name:, help:, type:, client:)
+      def initialize(name:, help:, type:, client:, opts: nil)
         @name = name
         @help = help
         @client = client
         @type = type
+        @opts = opts
       end
 
       def standard_values(value, keys, prometheus_exporter_action = nil)
@@ -25,6 +25,7 @@ module PrometheusExporter
           value: value
         }
         values[:prometheus_exporter_action] = prometheus_exporter_action if prometheus_exporter_action
+        values[:opts] = @opts if @opts
         values
       end
 
@@ -39,7 +40,6 @@ module PrometheusExporter
       def decrement(keys = nil, value = 1)
         @client.send_json(standard_values(value, keys, :decrement))
       end
-
     end
 
     def self.default
@@ -83,8 +83,8 @@ module PrometheusExporter
       @custom_labels = custom_labels
     end
 
-    def register(type, name, help)
-      metric = RemoteMetric.new(type: type, name: name, help: help, client: self)
+    def register(type, name, help, opts = nil)
+      metric = RemoteMetric.new(type: type, name: name, help: help, client: self, opts: opts)
       @metrics << metric
       metric
     end

data/lib/prometheus_exporter/instrumentation.rb

@@ -8,3 +8,4 @@ require_relative "instrumentation/delayed_job"
 require_relative "instrumentation/puma"
 require_relative "instrumentation/hutch"
 require_relative "instrumentation/unicorn"
+require_relative "instrumentation/active_record"

data/lib/prometheus_exporter/instrumentation/active_record.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+# collects stats from currently running process
+module PrometheusExporter::Instrumentation
+  class ActiveRecord
+    ALLOWED_CONFIG_LABELS = %i(database username host port)
+
+    def self.start(client: nil, frequency: 30, custom_labels: {}, config_labels: [])
+
+      # Not all rails versions support coonection pool stats
+      unless ::ActiveRecord::Base.connection_pool.respond_to?(:stat)
+        STDERR.puts("ActiveRecord connection pool stats not supported in your rails version")
+        return
+      end
+
+      config_labels.map!(&:to_sym)
+      validate_config_labels(config_labels)
+
+      active_record_collector = new(custom_labels, config_labels)
+
+      client ||= PrometheusExporter::Client.default
+
+      stop if @thread
+
+      @thread = Thread.new do
+        while true
+          begin
+            metrics = active_record_collector.collect
+            metrics.each { |metric| client.send_json metric }
+          rescue => e
+            STDERR.puts("Prometheus Exporter Failed To Collect Process Stats #{e}")
+          ensure
+            sleep frequency
+          end
+        end
+      end
+    end
+
+    def self.validate_config_labels(config_labels)
+      return if config_labels.size == 0
+      raise "Invalid Config Labels, available options #{ALLOWED_CONFIG_LABELS}" if (config_labels - ALLOWED_CONFIG_LABELS).size > 0
+    end
+
+    def self.stop
+      if t = @thread
+        t.kill
+        @thread = nil
+      end
+    end
+
+    def initialize(metric_labels, config_labels)
+      @metric_labels = metric_labels
+      @config_labels = config_labels
+      @hostname = nil
+    end
+
+    def hostname
+      @hostname ||=
+        begin
+          `hostname`.strip
+        rescue => e
+          STDERR.puts "Unable to lookup hostname #{e}"
+          "unknown-host"
+        end
+    end
+
+    def collect
+      metrics = []
+      collect_active_record_pool_stats(metrics)
+      metrics
+    end
+
+    def pid
+      @pid = ::Process.pid
+    end
+
+    def collect_active_record_pool_stats(metrics)
+      ObjectSpace.each_object(::ActiveRecord::ConnectionAdapters::ConnectionPool) do |pool|
+        next if pool.connections.nil?
+
+        labels_from_config = pool.spec.config
+          .select { |k, v| @config_labels.include? k }
+          .map { |k, v| [k.to_s.prepend("dbconfig_"), v] }
+
+        labels = @metric_labels.merge(pool_name: pool.spec.name).merge(Hash[labels_from_config])
+
+        metric = {
+          pid: pid,
+          type: "active_record",
+          hostname: hostname,
+          metric_labels: labels
+        }
+        metric.merge!(pool.stat)
+        metrics << metric
+      end
+    end
+  end
+end

data/lib/prometheus_exporter/instrumentation/delayed_job.rb

@@ -13,7 +13,9 @@ module PrometheusExporter::Instrumentation
           callbacks do |lifecycle|
             lifecycle.around(:invoke_job) do |job, *args, &block|
               max_attempts = Delayed::Worker.max_attempts
-              instrumenter.call(job, max_attempts, *args, &block)
+              enqueued_count = Delayed::Job.count
+              pending_count = Delayed::Job.where(attempts: 0, locked_at: nil).count
+              instrumenter.call(job, max_attempts, enqueued_count, pending_count, *args, &block)
             end
           end
         end
@@ -26,7 +28,7 @@ module PrometheusExporter::Instrumentation
       @client = client || PrometheusExporter::Client.default
     end
 
-    def call(job, max_attempts, *args, &block)
+    def call(job, max_attempts, enqueued_count, pending_count, *args, &block)
       success = false
       start = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
       attempts = job.attempts + 1 # Increment because we're adding the current attempt
@@ -42,7 +44,9 @@ module PrometheusExporter::Instrumentation
         success: success,
         duration: duration,
         attempts: attempts,
-        max_attempts: max_attempts
+        max_attempts: max_attempts,
+        enqueued: enqueued_count,
+        pending: pending_count
       )
     end
   end

data/lib/prometheus_exporter/instrumentation/unicorn.rb

@@ -9,7 +9,7 @@ end
 module PrometheusExporter::Instrumentation
   # collects stats from unicorn
   class Unicorn
-    def self.start(pid_file:, listener_address:, client:, frequency: 30)
+    def self.start(pid_file:, listener_address:, client: nil, frequency: 30)
       unicorn_collector = new(pid_file: pid_file, listener_address: listener_address)
       client ||= PrometheusExporter::Client.default
       Thread.new do

data/lib/prometheus_exporter/server.rb

@@ -13,3 +13,4 @@ require_relative "server/runner"
 require_relative "server/puma_collector"
 require_relative "server/hutch_collector"
 require_relative "server/unicorn_collector"
+require_relative "server/active_record_collector"

data/lib/prometheus_exporter/server/active_record_collector.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module PrometheusExporter::Server
+  class ActiveRecordCollector < TypeCollector
+    MAX_ACTIVERECORD_METRIC_AGE = 60
+    ACTIVE_RECORD_GAUGES = {
+      connections: "Total connections in pool",
+      busy: "Connections in use in pool",
+      dead: "Dead connections in pool",
+      idle: "Idle connections in pool",
+      waiting: "Connection requests waiting",
+      size: "Maximum allowed connection pool size"
+    }
+
+    def initialize
+      @active_record_metrics = []
+    end
+
+    def type
+      "active_record"
+    end
+
+    def metrics
+      return [] if @active_record_metrics.length == 0
+
+      metrics = {}
+
+      @active_record_metrics.map do |m|
+        metric_key = (m["metric_labels"] || {}).merge("pid" => m["pid"])
+
+        ACTIVE_RECORD_GAUGES.map do |k, help|
+          k = k.to_s
+          if v = m[k]
+            g = metrics[k] ||= PrometheusExporter::Metric::Gauge.new("active_record_connection_pool_#{k}", help)
+            g.observe(v, metric_key)
+          end
+        end
+      end
+
+      metrics.values
+    end
+
+    def collect(obj)
+      now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+
+      obj["created_at"] = now
+
+      @active_record_metrics.delete_if do |current|
+        (obj["pid"] == current["pid"] && obj["hostname"] == current["hostname"]) ||
+          (current["created_at"] + MAX_ACTIVERECORD_METRIC_AGE < now)
+      end
+
+      @active_record_metrics << obj
+    end
+  end
+end

data/lib/prometheus_exporter/server/collector.rb

@@ -17,6 +17,7 @@ module PrometheusExporter::Server
       register_collector(PumaCollector.new)
       register_collector(HutchCollector.new)
       register_collector(UnicornCollector.new)
+      register_collector(ActiveRecordCollector.new)
     end
 
     def register_collector(collector)
@@ -72,6 +73,7 @@ module PrometheusExporter::Server
     def register_metric_unsafe(obj)
       name = obj["name"]
       help = obj["help"]
+      opts = symbolize_keys(obj["opts"] || {})
 
       metric =
         case obj["type"]
@@ -80,9 +82,9 @@ module PrometheusExporter::Server
         when "counter"
           PrometheusExporter::Metric::Counter.new(name, help)
         when "summary"
-          PrometheusExporter::Metric::Summary.new(name, help)
+          PrometheusExporter::Metric::Summary.new(name, help, opts)
         when "histogram"
-          PrometheusExporter::Metric::Histogram.new(name, help)
+          PrometheusExporter::Metric::Histogram.new(name, help, opts)
         end
 
       if metric
@@ -91,5 +93,9 @@ module PrometheusExporter::Server
         STDERR.puts "failed to register metric #{obj}"
       end
     end
+
+    def symbolize_keys(hash)
+      hash.inject({}) { |memo, k| memo[k.first.to_sym] = k.last; memo }
+    end
   end
 end

data/lib/prometheus_exporter/server/delayed_job_collector.rb

@@ -21,12 +21,15 @@ module PrometheusExporter::Server
       @delayed_job_duration_seconds_summary.observe(obj["duration"], status: "success") if obj["success"]
       @delayed_job_duration_seconds_summary.observe(obj["duration"], status: "failed")  if !obj["success"]
       @delayed_job_attempts_summary.observe(obj["attempts"]) if obj["success"]
+      @delayed_jobs_enqueued.observe(obj["enqueued"])
+      @delayed_jobs_pending.observe(obj["pending"])
     end
 
     def metrics
       if @delayed_jobs_total
         [@delayed_job_duration_seconds, @delayed_jobs_total, @delayed_failed_jobs_total,
-         @delayed_jobs_max_attempts_reached_total, @delayed_job_duration_seconds_summary, @delayed_job_attempts_summary]
+         @delayed_jobs_max_attempts_reached_total, @delayed_job_duration_seconds_summary, @delayed_job_attempts_summary,
+         @delayed_jobs_enqueued, @delayed_jobs_pending]
       else
         []
       end
@@ -45,6 +48,14 @@ module PrometheusExporter::Server
         PrometheusExporter::Metric::Counter.new(
           "delayed_jobs_total", "Total number of delayed jobs executed.")
 
+        @delayed_jobs_enqueued =
+        PrometheusExporter::Metric::Gauge.new(
+          "delayed_jobs_enqueued", "Number of enqueued delayed jobs.")
+
+        @delayed_jobs_pending =
+        PrometheusExporter::Metric::Gauge.new(
+          "delayed_jobs_pending", "Number of pending delayed jobs.")
+
         @delayed_failed_jobs_total =
         PrometheusExporter::Metric::Counter.new(
           "delayed_failed_jobs_total", "Total number failed delayed jobs executed.")

data/lib/prometheus_exporter/server/process_collector.rb

@@ -35,6 +35,7 @@ module PrometheusExporter::Server
 
       @process_metrics.map do |m|
         metric_key = m["metric_labels"].merge("pid" => m["pid"])
+        metric_key.merge!(m["custom_labels"] || {})
 
         PROCESS_GAUGES.map do |k, help|
           k = k.to_s

data/lib/prometheus_exporter/server/unicorn_collector.rb

@@ -25,11 +25,13 @@ class PrometheusExporter::Server::UnicornCollector < PrometheusExporter::Server:
     metrics = {}
 
     @unicorn_metrics.map do |m|
+      labels = m["custom_labels"] || {}
+
       UNICORN_GAUGES.map do |k, help|
         k = k.to_s
         if (v = m[k])
           g = metrics[k] ||= PrometheusExporter::Metric::Gauge.new("unicorn_#{k}", help)
-          g.observe(v)
+          g.observe(v, labels)
         end
       end
     end

data/lib/prometheus_exporter/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PrometheusExporter
-  VERSION = "0.4.14"
+  VERSION = "0.4.15"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: prometheus_exporter
 version: !ruby/object:Gem::Version
-  version: 0.4.14
+  version: 0.4.15
 platform: ruby
 authors:
 - Sam Saffron
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-10-08 00:00:00.000000000 Z
+date: 2019-11-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rubocop
@@ -174,6 +174,7 @@ files:
 - lib/prometheus_exporter.rb
 - lib/prometheus_exporter/client.rb
 - lib/prometheus_exporter/instrumentation.rb
+- lib/prometheus_exporter/instrumentation/active_record.rb
 - lib/prometheus_exporter/instrumentation/delayed_job.rb
 - lib/prometheus_exporter/instrumentation/hutch.rb
 - lib/prometheus_exporter/instrumentation/method_profiler.rb
@@ -189,6 +190,7 @@ files:
 - lib/prometheus_exporter/metric/summary.rb
 - lib/prometheus_exporter/middleware.rb
 - lib/prometheus_exporter/server.rb
+- lib/prometheus_exporter/server/active_record_collector.rb
 - lib/prometheus_exporter/server/collector.rb
 - lib/prometheus_exporter/server/collector_base.rb
 - lib/prometheus_exporter/server/delayed_job_collector.rb