prometheus_exporter 0.4.9 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 71b406ba7c421d68e71912c260c0f520e31ee1f77eaf1fa1d5c1c2787f2de7d4
-  data.tar.gz: abdb9ec293106feb25d997887558229fe293f6530fb7799d03b94c6183f91a29
+  metadata.gz: c77ede3528c6008266620498f698c8a0e9e17bfa0c0691fedf5a3c4c3bdb8ebd
+  data.tar.gz: 62f8b159f68036f05b785fea4eebef48df66606b083136eda5c37e4d3c294132
 SHA512:
-  metadata.gz: b09926d30a09bd16df3afa1b55dddc37016d7ce93e4c42d54d874e6f04bc0b2f44ea8ee32211e09abe02ca1ee296bd02e0087640832d6f7854efc7ca300c8c32
-  data.tar.gz: 3c7d52250ce85c35a461959273b98b6d51db63ff33aa926a34b0c82dc447690dad66e557e7f0cde2bf3c0f99eaad615109c0ad8df7d8b81b36d0238a05515cab
+  metadata.gz: 2c0699014753102b514f1bd6fa965cd82f6d83c34084daf4a3b07537b9224295a1df02aedce3082588e44f4c001e71ea83178456bd7efd8a8eae836b2fc6e9a7
+  data.tar.gz: df738517c22422ba9e4e2e7cfd6ea6bbcf7542175a827b1632e7ff69af4651d4acf45bcec9693c4dd899b9b305b952fac989999ebea23013f3ddad0d3c396e00

data/.travis.yml

@@ -2,6 +2,11 @@ sudo: required
 language: ruby
 dist: trusty
 rvm:
-  - 2.5.0
+  - 2.3
+  - 2.4
+  - 2.5
+  - 2.6
 before_install: gem install bundler -v 1.16.1
 before_script: sudo sh -c 'echo 0 > /proc/sys/net/ipv6/conf/all/disable_ipv6'
+
+script: bundle exec rubocop && bundle exec rake

data/CHANGELOG

@@ -1,3 +1,46 @@
+0.5.0 - 14-02-2019
+
+- Breaking change: listen only to localhost by default to prevent unintended insecure configuration
+- FIX: Avoid calling `hostname` aggressively, instead cache it on the exporter instance
+
+0.4.17 - 13-01-2019
+
+- FEATURE: add support for `to_h` on all metrics which can be used to query existing key/values
+
+0.4.16 - 04-11-2019
+
+- FEATURE: Support #reset! on all metric types to reset a metric to default
+
+0.4.15 - 04-11-2019
+
+- FEATURE: Improve delayed job collector, add pending counts
+- FEATURE: New ActiveRecord collector (documented in readme)
+- FEATURE: Allow passing in histogram and summary options
+- FEATURE: Allow custom labels for unicorn collector
+
+0.4.14 - 10-09-2019
+
+- FEATURE: allow finding metrics by name RemoteMetric #find_registered_metric
+- FIX: guard socket closing
+
+0.4.13 - 09-07-2019
+
+- Fix: Memory leak in unicorn and puma collectors
+
+0.4.12 - 30-05-2019
+
+- Fix: unicorn collector reporting incorrect number of unicorn workers
+
+0.4.11 - 15-05-2019
+
+- Fix: Handle stopping nil worker_threads in Client
+- Dev: add frozen string literals
+
+0.4.10 - 29-04-2019
+
+- Fix: Custom label support for puma collector
+- Fix: Raindrops socket collector not working correctly
+
 0.4.9 - 11-04-2019
 
 - Fix: Gem was not working correctly in Ruby 2.4 and below due to a syntax error

data/Gemfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source "https://rubygems.org"
 
 git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }

data/Guardfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 guard :minitest do
   # with Minitest::Unit
   watch(%r{^test/(.*)\/?(.*)_test\.rb$})

data/README.md

@@ -13,10 +13,12 @@ To learn more see [Instrumenting Rails with Prometheus](https://samsaffron.com/a
   * [Rails integration](#rails-integration)
     * [Per-process stats](#per-process-stats)
     * [Sidekiq metrics](#sidekiq-metrics)
+    * [Shoryuken metrics](#shoryuken-metrics)
+    * [ActiveRecord Connection Pool Metrics](#activerecord-connection-pool-metrics)
     * [Delayed Job plugin](#delayed-job-plugin)
     * [Hutch metrics](#hutch-message-processing-tracer)
   * [Puma metrics](#puma-metrics)
-  * [Unicorn metrics](#unicorn-metrics)
+  * [Unicorn metrics](#unicorn-process-metrics)
   * [Custom type collectors](#custom-type-collectors)
   * [Multi process mode with custom collector](#multi-process-mode-with-custom-collector)
   * [GraphQL support](#graphql-support)
@@ -61,8 +63,9 @@ require 'prometheus_exporter/server'
 require 'prometheus_exporter/client'
 require 'prometheus_exporter/instrumentation'
 
+# bind is the address, on which the webserver will listen
 # port is the port that will provide the /metrics route
-server = PrometheusExporter::Server::WebServer.new port: 12345
+server = PrometheusExporter::Server::WebServer.new bind: 'localhost', port: 12345
 server.start
 
 # wire up a default local client
@@ -114,7 +117,7 @@ In some cases (for example, unicorn or puma clusters) you may want to aggregate
 
 Simplest way to achieve this is to use the built-in collector.
 
-First, run an exporter on your desired port (we use the default port of 9394):
+First, run an exporter on your desired port (we use the default bind to localhost and port of 9394):
 
 ```
 $ prometheus_exporter
@@ -148,6 +151,17 @@ awesome 10
 
 ```
 
+Custom quantiles for summaries and buckets for histograms can also be passed in.
+
+```ruby
+require 'prometheus_exporter/client'
+
+client = PrometheusExporter::Client.default
+histogram = client.register(:histogram, "api_time", "time to call api", buckets: [0.1, 0.5, 1])
+
+histogram.observe(0.2, api: 'twitter')
+```
+
 ### Rails integration
 
 You can easily integrate into any Rack application.
@@ -175,6 +189,60 @@ Ensure you run the exporter in a monitored background process:
 $ bundle exec prometheus_exporter
 ```
 
+#### Activerecord Connection Pool Metrics
+
+This collects activerecord connection pool metrics.
+
+It supports injection of custom labels and the connection config options (`username`, `database`, `host`, `port`) as labels.
+
+For Puma single mode 
+```ruby
+#in puma.rb
+require 'prometheus_exporter/instrumentation'
+PrometheusExporter::Instrumentation::ActiveRecord.start(
+  custom_labels: { type: "puma_single_mode" }, #optional params
+  config_labels: [:database, :host] #optional params
+)
+```
+
+For Puma cluster mode
+
+```ruby
+# in puma.rb
+on_worker_boot do
+  require 'prometheus_exporter/instrumentation'
+  PrometheusExporter::Instrumentation::ActiveRecord.start(
+    custom_labels: { type: "puma_worker" }, #optional params
+    config_labels: [:database, :host] #optional params
+  )
+end
+```
+
+For Unicorn / Passenger
+
+```ruby
+after_fork do |_server, _worker|
+  require 'prometheus_exporter/instrumentation'
+  PrometheusExporter::Instrumentation::ActiveRecord.start(
+    custom_labels: { type: "unicorn_worker" }, #optional params
+    config_labels: [:database, :host] #optional params
+  )
+end
+```
+
+For Sidekiq
+```ruby
+Sidekiq.configure_server do |config|
+  config.on :startup do
+    require 'prometheus_exporter/instrumentation'
+    PrometheusExporter::Instrumentation::ActiveRecord.start(
+      custom_labels: { type: "sidekiq" }, #optional params
+      config_labels: [:database, :host] #optional params
+    )
+  end
+end
+```
+
 #### Per-process stats
 
 You may also be interested in per-process stats. This collects memory and GC stats:
@@ -231,6 +299,19 @@ Sometimes the Sidekiq server shuts down before it can send metrics, that were ge
   end
 ```
 
+#### Shoryuken metrics
+
+For Shoryuken metrics (how many jobs ran? how many failed? how long did they take? how many were restarted?)
+
+```ruby
+Shoryuken.configure_server do |config|
+  config.server_middleware do |chain|
+    require 'prometheus_exporter/instrumentation'
+    chain.add PrometheusExporter::Instrumentation::Shoryuken
+  end
+end
+```
+
 #### Delayed Job plugin
 
 In an initializer:

data/Rakefile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "bundler/gem_tasks"
 require "rake/testtask"
 

data/bench/bench.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative '../lib/prometheus_exporter'
 require_relative '../lib/prometheus_exporter/client'
 require_relative '../lib/prometheus_exporter/server'

data/bin/prometheus_exporter

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
 require 'optparse'
 
@@ -18,6 +19,12 @@ def run
            "Port exporter should listen on (default: #{PrometheusExporter::DEFAULT_PORT})") do |o|
       options[:port] = o.to_i
     end
+    opt.on('-b',
+           '--bind STRING',
+           String,
+           "IP address exporter should listen on (default: #{PrometheusExporter::DEFAULT_BIND_ADDRESS})") do |o|
+      options[:bind] = o.to_s
+    end
     opt.on('-t',
            '--timeout INTEGER',
            Integer,
@@ -80,7 +87,7 @@ def run
 
   runner = PrometheusExporter::Server::Runner.new(options)
 
-  puts "#{Time.now} Starting prometheus exporter on port #{runner.port}"
+  puts "#{Time.now} Starting prometheus exporter on #{runner.bind}:#{runner.port}"
   runner.start
   sleep
 end

data/examples/custom_collector.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class MyCustomCollector < PrometheusExporter::Server::Collector
   def initialize
     @gauge1 = PrometheusExporter::Metric::Gauge.new("thing1", "I am thing 1")

data/lib/prometheus_exporter/client.rb

@@ -4,14 +4,16 @@ require 'socket'
 require 'thread'
 
 module PrometheusExporter
-
   class Client
     class RemoteMetric
-      def initialize(name:, help:, type:, client:)
+      attr_reader :name, :type, :help
+
+      def initialize(name:, help:, type:, client:, opts: nil)
         @name = name
         @help = help
         @client = client
         @type = type
+        @opts = opts
       end
 
       def standard_values(value, keys, prometheus_exporter_action = nil)
@@ -23,6 +25,7 @@ module PrometheusExporter
           value: value
         }
         values[:prometheus_exporter_action] = prometheus_exporter_action if prometheus_exporter_action
+        values[:opts] = @opts if @opts
         values
       end
 
@@ -37,7 +40,6 @@ module PrometheusExporter
       def decrement(keys = nil, value = 1)
         @client.send_json(standard_values(value, keys, :decrement))
       end
-
     end
 
     def self.default
@@ -81,12 +83,22 @@ module PrometheusExporter
       @custom_labels = custom_labels
     end
 
-    def register(type, name, help)
-      metric = RemoteMetric.new(type: type, name: name, help: help, client: self)
+    def register(type, name, help, opts = nil)
+      metric = RemoteMetric.new(type: type, name: name, help: help, client: self, opts: opts)
       @metrics << metric
       metric
     end
 
+    def find_registered_metric(name, type: nil, help: nil)
+      @metrics.find do |metric|
+        type_match = type ? metric.type == type : true
+        help_match = help ? metric.help == help : true
+        name_match = metric.name == name
+
+        type_match && help_match && name_match
+      end
+    end
+
     def send_json(obj)
       payload = @custom_labels.nil? ? obj : obj.merge(custom_labels: @custom_labels)
       send(@json_serializer.dump(payload))
@@ -124,12 +136,12 @@ module PrometheusExporter
       @mutex.synchronize do
         wait_for_empty_queue_with_timeout(wait_timeout_seconds)
         @worker_thread&.kill
-        while @worker_thread.alive?
+        while @worker_thread&.alive?
           sleep 0.001
         end
         @worker_thread = nil
-      end
         close_socket!
+      end
     end
 
     private

data/lib/prometheus_exporter/instrumentation/active_record.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+# collects stats from currently running process
+module PrometheusExporter::Instrumentation
+  class ActiveRecord
+    ALLOWED_CONFIG_LABELS = %i(database username host port)
+
+    def self.start(client: nil, frequency: 30, custom_labels: {}, config_labels: [])
+
+      # Not all rails versions support coonection pool stats
+      unless ::ActiveRecord::Base.connection_pool.respond_to?(:stat)
+        STDERR.puts("ActiveRecord connection pool stats not supported in your rails version")
+        return
+      end
+
+      config_labels.map!(&:to_sym)
+      validate_config_labels(config_labels)
+
+      active_record_collector = new(custom_labels, config_labels)
+
+      client ||= PrometheusExporter::Client.default
+
+      stop if @thread
+
+      @thread = Thread.new do
+        while true
+          begin
+            metrics = active_record_collector.collect
+            metrics.each { |metric| client.send_json metric }
+          rescue => e
+            STDERR.puts("Prometheus Exporter Failed To Collect Process Stats #{e}")
+          ensure
+            sleep frequency
+          end
+        end
+      end
+    end
+
+    def self.validate_config_labels(config_labels)
+      return if config_labels.size == 0
+      raise "Invalid Config Labels, available options #{ALLOWED_CONFIG_LABELS}" if (config_labels - ALLOWED_CONFIG_LABELS).size > 0
+    end
+
+    def self.stop
+      if t = @thread
+        t.kill
+        @thread = nil
+      end
+    end
+
+    def initialize(metric_labels, config_labels)
+      @metric_labels = metric_labels
+      @config_labels = config_labels
+    end
+
+    def collect
+      metrics = []
+      collect_active_record_pool_stats(metrics)
+      metrics
+    end
+
+    def pid
+      @pid = ::Process.pid
+    end
+
+    def collect_active_record_pool_stats(metrics)
+      ObjectSpace.each_object(::ActiveRecord::ConnectionAdapters::ConnectionPool) do |pool|
+        next if pool.connections.nil?
+
+        labels_from_config = pool.spec.config
+          .select { |k, v| @config_labels.include? k }
+          .map { |k, v| [k.to_s.prepend("dbconfig_"), v] }
+
+        labels = @metric_labels.merge(pool_name: pool.spec.name).merge(Hash[labels_from_config])
+
+        metric = {
+          pid: pid,
+          type: "active_record",
+          hostname: ::PrometheusExporter.hostname,
+          metric_labels: labels
+        }
+        metric.merge!(pool.stat)
+        metrics << metric
+      end
+    end
+  end
+end

data/lib/prometheus_exporter/instrumentation/delayed_job.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module PrometheusExporter::Instrumentation
   class DelayedJob
     JOB_CLASS_REGEXP = %r{job_class: (\w+:{0,2})+}.freeze
@@ -11,7 +13,9 @@ module PrometheusExporter::Instrumentation
           callbacks do |lifecycle|
             lifecycle.around(:invoke_job) do |job, *args, &block|
               max_attempts = Delayed::Worker.max_attempts
-              instrumenter.call(job, max_attempts, *args, &block)
+              enqueued_count = Delayed::Job.count
+              pending_count = Delayed::Job.where(attempts: 0, locked_at: nil).count
+              instrumenter.call(job, max_attempts, enqueued_count, pending_count, *args, &block)
             end
           end
         end
@@ -24,7 +28,7 @@ module PrometheusExporter::Instrumentation
       @client = client || PrometheusExporter::Client.default
     end
 
-    def call(job, max_attempts, *args, &block)
+    def call(job, max_attempts, enqueued_count, pending_count, *args, &block)
       success = false
       start = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
       attempts = job.attempts + 1 # Increment because we're adding the current attempt
@@ -40,7 +44,9 @@ module PrometheusExporter::Instrumentation
         success: success,
         duration: duration,
         attempts: attempts,
-        max_attempts: max_attempts
+        max_attempts: max_attempts,
+        enqueued: enqueued_count,
+        pending: pending_count
       )
     end
   end

data/lib/prometheus_exporter/instrumentation/method_profiler.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # see https://samsaffron.com/archive/2017/10/18/fastest-way-to-profile-a-method-in-ruby
 module PrometheusExporter::Instrumentation; end
 

data/lib/prometheus_exporter/instrumentation/process.rb

@@ -42,24 +42,13 @@ module PrometheusExporter::Instrumentation
 
     def initialize(metric_labels)
       @metric_labels = metric_labels
-      @hostname = nil
-    end
-
-    def hostname
-      @hostname ||=
-        begin
-          `hostname`.strip
-        rescue => e
-          STDERR.puts "Unable to lookup hostname #{e}"
-          "unknown-host"
-        end
     end
 
     def collect
       metric = {}
       metric[:type] = "process"
       metric[:metric_labels] = @metric_labels
-      metric[:hostname] = hostname
+      metric[:hostname] = ::PrometheusExporter.hostname
       collect_gc_stats(metric)
       collect_v8_stats(metric)
       collect_process_stats(metric)

data/lib/prometheus_exporter/instrumentation/puma.rb

@@ -1,4 +1,6 @@
-require 'json'
+# frozen_string_literal: true
+
+require "json"
 
 # collects stats from puma
 module PrometheusExporter::Instrumentation
@@ -30,7 +32,7 @@ module PrometheusExporter::Instrumentation
     def collect_puma_stats(metric)
       stats = JSON.parse(::Puma.stats)
 
-      if stats.key? 'workers'
+      if stats.key?("workers")
         metric[:phase] = stats["phase"]
         metric[:workers_total] = stats["workers"]
         metric[:booted_workers_total] = stats["booted_workers"]

data/lib/prometheus_exporter/instrumentation/shoryuken.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module PrometheusExporter::Instrumentation
+  class Shoryuken
+
+    def initialize(client: nil)
+      @client = client || PrometheusExporter::Client.default
+    end
+
+    def call(worker, queue, msg, body)
+      success = false
+      start = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      result = yield
+      success = true
+      result
+    rescue ::Shoryuken::Shutdown => e
+      shutdown = true
+      raise e
+    ensure
+      duration = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - start
+      @client.send_json(
+          type: "shoryuken",
+          queue: queue,
+          name: worker.class.name,
+          success: success,
+          shutdown: shutdown,
+          duration: duration
+      )
+    end
+  end
+end

data/lib/prometheus_exporter/instrumentation/sidekiq.rb

@@ -4,11 +4,15 @@ module PrometheusExporter::Instrumentation
   class Sidekiq
     def self.death_handler
       -> (job, ex) do
-        PrometheusExporter::Client.default.send_json(
-          type: "sidekiq",
-          name: job["class"],
-          dead: true,
-        )
+        job_is_fire_and_forget = job["retry"] == false
+
+        unless job_is_fire_and_forget
+          PrometheusExporter::Client.default.send_json(
+            type: "sidekiq",
+            name: job["class"],
+            dead: true,
+          )
+        end
       end
     end
 

data/lib/prometheus_exporter/instrumentation/unicorn.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 begin
   require 'raindrops'
 rescue LoadError
@@ -7,7 +9,7 @@ end
 module PrometheusExporter::Instrumentation
   # collects stats from unicorn
   class Unicorn
-    def self.start(pid_file:, listener_address:, client:, frequency: 30)
+    def self.start(pid_file:, listener_address:, client: nil, frequency: 30)
       unicorn_collector = new(pid_file: pid_file, listener_address: listener_address)
       client ||= PrometheusExporter::Client.default
       Thread.new do
@@ -49,21 +51,21 @@ module PrometheusExporter::Instrumentation
 
     def worker_process_count
       return nil unless File.exist?(@pid_file)
-      pid = File.read(@pid_file)
+      pid = File.read(@pid_file).to_i
 
-      return nil unless pid && pid.to_i > 0
+      return nil if pid < 1
 
       # find all processes whose parent is the unicorn master
       # but we're actually only interested in the number of processes (= lines of output)
-      result = `ps --no-header -o pid --ppid #{pid}`
+      result = `pgrep -P #{pid} -f unicorn -a`
       result.lines.count
     end
 
     def listener_address_stats
       if @tcp
-        Raindrops::Linux.tcp_listener_stats(@listener_address)[@listener_address]
+        Raindrops::Linux.tcp_listener_stats([@listener_address])[@listener_address]
       else
-        Raindrops::Linux.unix_listener_stats(@listener_address)[@listener_address]
+        Raindrops::Linux.unix_listener_stats([@listener_address])[@listener_address]
       end
     end
   end

data/lib/prometheus_exporter/instrumentation.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+
+require_relative "client"
 require_relative "instrumentation/process"
 require_relative "instrumentation/method_profiler"
 require_relative "instrumentation/sidekiq"
@@ -5,3 +8,5 @@ require_relative "instrumentation/delayed_job"
 require_relative "instrumentation/puma"
 require_relative "instrumentation/hutch"
 require_relative "instrumentation/unicorn"
+require_relative "instrumentation/active_record"
+require_relative "instrumentation/shoryuken"

data/lib/prometheus_exporter/metric/base.rb

@@ -34,6 +34,14 @@ module PrometheusExporter::Metric
       raise "Not implemented"
     end
 
+    def reset!
+      raise "Not implemented"
+    end
+
+    def to_h
+      raise "Not implemented"
+    end
+
     def from_json(json)
       json = JSON.parse(json) if String === json
       @name = json["name"]

data/lib/prometheus_exporter/metric/counter.rb

@@ -6,19 +6,27 @@ module PrometheusExporter::Metric
 
     def initialize(name, help)
       super
-      @data = {}
+      reset!
     end
 
     def type
       "counter"
     end
 
+    def reset!
+      @data = {}
+    end
+
     def metric_text
       @data.map do |labels, value|
         "#{prefix(@name)}#{labels_text(labels)} #{value}"
       end.join("\n")
     end
 
+    def to_h
+      @data.dup
+    end
+
     def observe(increment = 1, labels = {})
       @data[labels] ||= 0
       @data[labels] += increment