prometheus_exporter 0.4.16 → 0.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f71a3c031cd843aba70a3fb4692b73507c7ad7b1256e1cf5b1dbb264cebf73d3
-  data.tar.gz: 0fbc4a8c24a2ce6d95e6aa315938fc0f4be46c8e769db49397fcbfd2b3dd81f4
+  metadata.gz: a284b77cbd17e80457e94520d18976dc3dd6fb8c238d8a90cda6513e291f01ad
+  data.tar.gz: 4464419fc9c67d387edd9d6d5bdfac1411d0864d29c0cd3a6f10d2f02b3d7e24
 SHA512:
-  metadata.gz: 760279a05df24ae8b06d11bed36d4121afe223020bd36a4cd8f945195c402313aea6c58c81b1f85c0d40b64812af31a37e1537ecc14573b435830311c0415cfd
-  data.tar.gz: cc36f5859f8195000899e8b2bb066d9bf03698a8280e532231fda9e4bd7756d478fb0d5bd7c4e87dd34bb0a73f88485b198fd53f8c8bf8709c761fdcce41347c
+  metadata.gz: 84c2e2af95a8839cd9deecca9fbc26bacff590ae5cb3677862f1eb1d476401114406130cc78d2248cec0380e70c1b74b306e1265b3343bbac9c09d8400e692a3
+  data.tar.gz: 41898546588ef7eafa772252ef5ed1a8c222e4009e64ccec35d463217336597a94f1c844f3e2174e030c19a7c1b90d1028e5729cc393cf797654362f5d66b7f2

data/.rubocop.yml

@@ -1 +1,2 @@
-inherit_from: https://raw.githubusercontent.com/discourse/discourse/master/.rubocop.yml
+inherit_gem:
+  rubocop-discourse: default.yml

data/CHANGELOG

@@ -1,3 +1,25 @@
+0.5.3 - 29-07-2020
+
+- FEATURE: added #remove to all metric types so users can remove specific labels if needed
+
+0.5.2 - 01-07-2020
+
+- FEATURE: expanded instrumentation for sidekiq
+- FEATURE: configurable default labels
+
+0.5.1 - 25-02-2020
+
+- FEATURE: Allow configuring the default client's host and port via environment variables
+
+0.5.0 - 14-02-2020
+
+- Breaking change: listen only to localhost by default to prevent unintended insecure configuration
+- FIX: Avoid calling `hostname` aggressively, instead cache it on the exporter instance
+
+0.4.17 - 13-01-2020
+
+- FEATURE: add support for `to_h` on all metrics which can be used to query existing key/values
+
 0.4.16 - 04-11-2019
 
 - FEATURE: Support #reset! on all metric types to reset a metric to default

data/README.md

@@ -13,6 +13,7 @@ To learn more see [Instrumenting Rails with Prometheus](https://samsaffron.com/a
   * [Rails integration](#rails-integration)
     * [Per-process stats](#per-process-stats)
     * [Sidekiq metrics](#sidekiq-metrics)
+    * [Shoryuken metrics](#shoryuken-metrics)
     * [ActiveRecord Connection Pool Metrics](#activerecord-connection-pool-metrics)
     * [Delayed Job plugin](#delayed-job-plugin)
     * [Hutch metrics](#hutch-message-processing-tracer)
@@ -23,6 +24,7 @@ To learn more see [Instrumenting Rails with Prometheus](https://samsaffron.com/a
   * [GraphQL support](#graphql-support)
   * [Metrics default prefix / labels](#metrics-default-prefix--labels)
   * [Client default labels](#client-default-labels)
+  * [Client default host](#client-default-host)
 * [Transport concerns](#transport-concerns)
 * [JSON generation and parsing](#json-generation-and-parsing)
 * [Contributing](#contributing)
@@ -62,8 +64,9 @@ require 'prometheus_exporter/server'
 require 'prometheus_exporter/client'
 require 'prometheus_exporter/instrumentation'
 
+# bind is the address, on which the webserver will listen
 # port is the port that will provide the /metrics route
-server = PrometheusExporter::Server::WebServer.new port: 12345
+server = PrometheusExporter::Server::WebServer.new bind: 'localhost', port: 12345
 server.start
 
 # wire up a default local client
@@ -115,7 +118,7 @@ In some cases (for example, unicorn or puma clusters) you may want to aggregate
 
 Simplest way to achieve this is to use the built-in collector.
 
-First, run an exporter on your desired port (we use the default port of 9394):
+First, run an exporter on your desired port (we use the default bind to localhost and port of 9394):
 
 ```
 $ prometheus_exporter
@@ -187,13 +190,30 @@ Ensure you run the exporter in a monitored background process:
 $ bundle exec prometheus_exporter
 ```
 
+#### Metrics collected by Rails integration middleware
+
+| Type    | Name                            | Description                                                 |
+| ---     | ---                             | ---                                                         |
+| Counter | `http_requests_total`           | Total HTTP requests from web app                            |
+| Summary | `http_duration_seconds`         | Time spent in HTTP reqs in seconds                          |
+| Summary | `http_redis_duration_seconds`¹  | Time spent in HTTP reqs in Redis, in seconds                |
+| Summary | `http_sql_duration_seconds`²    | Time spent in HTTP reqs in SQL in seconds                   |
+| Summary | `http_queue_duration_seconds`³  | Time spent queueing the request in load balancer in seconds |
+
+All metrics have a `controller` and an `action` label.  
+`http_requests_total` additionally has a (HTTP response) `status` label.  
+
+¹) Only available when Redis is used.  
+²) Only available when Mysql or PostgreSQL are used.  
+³) Only available when [Instrumenting Request Queueing Time](#instrumenting-request-queueing-time) is set up.  
+
 #### Activerecord Connection Pool Metrics
 
 This collects activerecord connection pool metrics.
 
 It supports injection of custom labels and the connection config options (`username`, `database`, `host`, `port`) as labels.
 
-For Puma single mode 
+For Puma single mode
 ```ruby
 #in puma.rb
 require 'prometheus_exporter/instrumentation'
@@ -219,7 +239,7 @@ end
 For Unicorn / Passenger
 
 ```ruby
-after_fork do 
+after_fork do |_server, _worker|
   require 'prometheus_exporter/instrumentation'
   PrometheusExporter::Instrumentation::ActiveRecord.start(
     custom_labels: { type: "unicorn_worker" }, #optional params
@@ -241,6 +261,19 @@ Sidekiq.configure_server do |config|
 end
 ```
 
+##### Metrics collected by ActiveRecord Instrumentation
+
+| Type  | Name                                        | Description                           |
+| ---   | ---                                         | ---                                   |
+| Gauge | `active_record_connection_pool_connections` | Total connections in pool             |
+| Gauge | `active_record_connection_pool_busy`        | Connections in use in pool            |
+| Gauge | `active_record_connection_pool_dead`        | Dead connections in pool              |
+| Gauge | `active_record_connection_pool_idle`        | Idle connections in pool              |
+| Gauge | `active_record_connection_pool_waiting`     | Connection requests waiting           |
+| Gauge | `active_record_connection_pool_size`        | Maximum allowed connection pool size  |
+
+All metrics collected by the ActiveRecord integration include at least the following labels: `pid` (of the process the stats where collected in), `pool_name`, any labels included in the `config_labels` option (prefixed with `dbconfig_`, example: `dbconfig_host`), and all custom labels provided with the `custom_labels` option.
+
 #### Per-process stats
 
 You may also be interested in per-process stats. This collects memory and GC stats:
@@ -257,11 +290,30 @@ end
 # in unicorn/puma/passenger be sure to run a new process instrumenter after fork
 after_fork do
   require 'prometheus_exporter/instrumentation'
-  PrometheusExporter::Instrumentation::Process.start(type:"web")
+  PrometheusExporter::Instrumentation::Process.start(type: "web")
 end
 
 ```
 
+##### Metrics collected by Process Instrumentation
+
+| Type    | Name                      | Description                                  |
+| ---     | ---                       | ---                                          |
+| Gauge   | `heap_free_slots`         | Free ruby heap slots                         |
+| Gauge   | `heap_live_slots`         | Used ruby heap slots                         |
+| Gauge   | `v8_heap_size`*           | Total JavaScript V8 heap size (bytes)        |
+| Gauge   | `v8_used_heap_size`*      | Total used JavaScript V8 heap size (bytes)   |
+| Gauge   | `v8_physical_size`*       | Physical size consumed by V8 heaps           |
+| Gauge   | `v8_heap_count`*          | Number of V8 contexts running                |
+| Gauge   | `rss`                     | Total RSS used by process                    |
+| Counter | `major_gc_ops_total`      | Major GC operations by process               |
+| Counter | `minor_gc_ops_total`      | Minor GC operations by process               |
+| Counter | `allocated_objects_total` | Total number of allocated objects by process |
+
+_Metrics marked with * are only collected when `MiniRacer` is defined._
+
+Metrics collected by Process instrumentation include labels `type` (as given with the `type` option), `pid` (of the process the stats where collected in), and any custom labels given to `Process.start` with the `labels` option.
+
 #### Sidekiq metrics
 
 Including Sidekiq metrics (how many jobs ran? how many failed? how long did they take? how many are dead? how many were restarted?)
@@ -276,6 +328,17 @@ Sidekiq.configure_server do |config|
 end
 ```
 
+To monitor Queue size and latency:
+
+```ruby
+Sidekiq.configure_server do |config|
+  config.on :startup do
+    require 'prometheus_exporter/instrumentation'
+    PrometheusExporter::Instrumentation::SidekiqQueue.start
+  end
+end
+```
+
 To monitor Sidekiq process info:
 
 ```ruby
@@ -297,6 +360,59 @@ Sometimes the Sidekiq server shuts down before it can send metrics, that were ge
   end
 ```
 
+##### Metrics collected by Sidekiq Instrumentation
+
+**PrometheusExporter::Instrumentation::Sidekiq**
+| Type    | Name                           | Description                                                                  |
+| ---     | ---                            | ---                                                                          |
+| Counter | `sidekiq_job_duration_seconds` | Total time spent in sidekiq jobs                                             |
+| Counter | `sidekiq_jobs_total`           | Total number of sidekiq jobs executed                                        |
+| Counter | `sidekiq_restarted_jobs_total` | Total number of sidekiq jobs that we restarted because of a sidekiq shutdown |
+| Counter | `sidekiq_failed_jobs_total`    | Total number of failed sidekiq jobs                                          |
+
+All metrics have a `job_name` label.
+
+**PrometheusExporter::Instrumentation::Sidekiq.death_handler**
+| Type    | Name                      | Description                       |
+| ---     | ---                       | ---                               |
+| Counter | `sidekiq_dead_jobs_total` | Total number of dead sidekiq jobs |
+
+This metric also has a `job_name` label.
+
+**PrometheusExporter::Instrumentation::SidekiqQueue**
+| Type  | Name                            | Description                  |
+| ---   | ---                             | ---                          |
+| Gauge | `sidekiq_queue_backlog_total`   | Size of the sidekiq queue    |
+| Gauge | `sidekiq_queue_latency_seconds` | Latency of the sidekiq queue |
+
+Both metrics will have a `queue` label with the name of the queue.
+
+_See [Metrics collected by Process Instrumentation](#metrics-collected-by-process-instrumentation) for a list of metrics the Process instrumentation will produce._  
+
+#### Shoryuken metrics
+
+For Shoryuken metrics (how many jobs ran? how many failed? how long did they take? how many were restarted?)
+
+```ruby
+Shoryuken.configure_server do |config|
+  config.server_middleware do |chain|
+    require 'prometheus_exporter/instrumentation'
+    chain.add PrometheusExporter::Instrumentation::Shoryuken
+  end
+end
+```
+
+##### Metrics collected by Shoryuken Instrumentation
+
+| Type    | Name                             | Description                                                                      |
+| ---     | ---                              | ---                                                                              |
+| Counter | `shoryuken_job_duration_seconds` | Total time spent in shoryuken jobs                                               |
+| Counter | `shoryuken_jobs_total`           | Total number of shoryuken jobs executed                                          |
+| Counter | `shoryuken_restarted_jobs_total` | Total number of shoryuken jobs that we restarted because of a shoryuken shutdown |
+| Counter | `shoryuken_failed_jobs_total`    | Total number of failed shoryuken jobs                                            |
+
+All metrics have labels for `job_name` and `queue_name`.  
+
 #### Delayed Job plugin
 
 In an initializer:
@@ -308,6 +424,19 @@ unless Rails.env == "test"
 end
 ```
 
+##### Metrics collected by Delayed Job Instrumentation
+
+| Type    | Name                                      | Description                                                        | Labels     |
+| ---     | ---                                       | ---                                                                | ---        |
+| Counter | `delayed_job_duration_seconds`            | Total time spent in delayed jobs                                   | `job_name` |
+| Counter | `delayed_jobs_total`                      | Total number of delayed jobs executed                              | `job_name` |
+| Gauge   | `delayed_jobs_enqueued`                   | Number of enqueued delayed jobs                                    | -          |
+| Gauge   | `delayed_jobs_pending`                    | Number of pending delayed jobs                                     | -          |
+| Counter | `delayed_failed_jobs_total`               | Total number failed delayed jobs executed                          | `job_name` |
+| Counter | `delayed_jobs_max_attempts_reached_total` | Total number of delayed jobs that reached max attempts             | -          |
+| Summary | `delayed_job_duration_seconds_summary`    | Summary of the time it takes jobs to execute                       | `status`   |
+| Summary | `delayed_job_attempts_summary`            | Summary of the amount of attempts it takes delayed jobs to succeed | -          |
+
 #### Hutch Message Processing Tracer
 
 Capture [Hutch](https://github.com/gocardless/hutch) metrics (how many jobs ran? how many failed? how long did they take?)
@@ -319,6 +448,16 @@ unless Rails.env == "test"
 end
 ```
 
+##### Metrics collected by Hutch Instrumentation
+
+| Type    | Name                         | Description                             |
+| ---     | ---                          | ---                                     |
+| Counter | `hutch_job_duration_seconds` | Total time spent in hutch jobs          |
+| Counter | `hutch_jobs_total`           | Total number of hutch jobs executed     |
+| Counter | `hutch_failed_jobs_total`    | Total number failed hutch jobs executed |
+
+All metrics have a `job_name` label. 
+
 #### Instrumenting Request Queueing Time
 
 Request Queueing is defined as the time it takes for a request to reach your application (instrumented by this `prometheus_exporter`) from farther upstream (as your load balancer). A high queueing time usually means that your backend cannot handle all the incoming requests in time, so they queue up (= you should see if you need to add more capacity).
@@ -343,6 +482,20 @@ after_worker_boot do
 end
 ```
 
+#### Metrics collected by Puma Instrumentation
+
+| Type  | Name                              | Description                                                 |
+| ---   | ---                               | ---                                                         |
+| Gauge | `puma_workers_total`              | Number of puma workers                                      |
+| Gauge | `puma_booted_workers_total`       | Number of puma workers booted                               |
+| Gauge | `puma_old_workers_total`          | Number of old puma workers                                  |
+| Gauge | `puma_running_threads_total`      | Number of puma threads currently running                    |
+| Gauge | `puma_request_backlog_total`      | Number of requests waiting to be processed by a puma thread |
+| Gauge | `puma_thread_pool_capacity_total` | Number of puma threads available at current scale           |
+| Gauge | `puma_max_threads_total`          | Number of puma threads at available at max scale            |
+
+All metrics may have a `phase` label.   
+
 ### Unicorn process metrics
 
 In order to gather metrics from unicorn processes, we use `rainbows`, which exposes `Rainbows::Linux.tcp_listener_stats` to gather information about active workers and queued requests. To start monitoring your unicorn processes, you'll need to know both the path to unicorn PID file and the listen address (`pid_file` and `listen` in your unicorn config file)
@@ -358,6 +511,14 @@ prometheus_exporter --unicorn-master /var/run/unicorn.pid --unicorn-listen-addre
 
 Note: You must install the `raindrops` gem in your `Gemfile` or locally.
 
+#### Metrics collected by Unicorn Instrumentation
+
+| Type  | Name                            | Description                                                    |
+| ---   | ---                             | ---                                                            |
+| Gauge | `unicorn_workers_total`         | Number of unicorn workers                                      |
+| Gauge | `unicorn_active_workers_total`  | Number of active unicorn workers                               |
+| Gauge | `unicorn_request_backlog_total` | Number of requests waiting to be processed by a unicorn worker |
+
 ### Custom type collectors
 
 In some cases you may have custom metrics you want to ship the collector in a batch. In this case you may still be interested in the base collector behavior, but would like to add your own special messages.
@@ -526,6 +687,26 @@ ruby_web_requests{hostname="app-server-01",route="test/route"} 1
 ruby_web_requests{hostname="app-server-01"} 1
 ```
 
+### Exporter Process Configuration
+
+When running the process for `prometheus_exporter` using `bin/prometheus_exporter`, there are several configurations that
+can be passed in:
+
+The following will run the process at
+- Port `8080` (default `9394`)
+- Bind to `0.0.0.0` (default `localhost`)
+- Timeout in `1 second` for metrics endpoint (default `2 seconds`)
+- Metric prefix as `foo_` (default `ruby_`)
+- Default labels as `{environment: "integration", foo: "bar"}`
+
+```bash
+prometheus_exporter -p 8080 \
+                    -b 0.0.0.0 \
+                    -t 1 \
+                    --label '{"environment": "integration", "foo": "bar"}' \
+                    --prefix 'foo_'
+```
+
 ### Client default labels
 
 You can specify a default label for instrumentation metrics sent by a specific client. For example:
@@ -545,6 +726,9 @@ Will result in:
 http_requests_total{controller="home","action"="index",service="app-server-01",app_name="app-01"} 2
 http_requests_total{service="app-server-01",app_name="app-01"} 1
 ```
+### Client default host
+
+By default, `PrometheusExporter::Client.default` connects to `localhost:9394`. If your setup requires this (e.g. when using `docker-compose`), you can change the default host and port by setting the environment variables `PROMETHEUS_EXPORTER_HOST` and `PROMETHEUS_EXPORTER_PORT`.
 
 ## Transport concerns
 

data/bin/prometheus_exporter

@@ -2,6 +2,7 @@
 # frozen_string_literal: true
 
 require 'optparse'
+require 'json'
 
 require_relative "./../lib/prometheus_exporter"
 require_relative "./../lib/prometheus_exporter/server"
@@ -19,6 +20,12 @@ def run
            "Port exporter should listen on (default: #{PrometheusExporter::DEFAULT_PORT})") do |o|
       options[:port] = o.to_i
     end
+    opt.on('-b',
+           '--bind STRING',
+           String,
+           "IP address exporter should listen on (default: #{PrometheusExporter::DEFAULT_BIND_ADDRESS})") do |o|
+      options[:bind] = o.to_s
+    end
     opt.on('-t',
            '--timeout INTEGER',
            Integer,
@@ -28,6 +35,9 @@ def run
     opt.on('--prefix METRIC_PREFIX', "Prefix to apply to all metrics (default: #{PrometheusExporter::DEFAULT_PREFIX})") do |o|
       options[:prefix] = o.to_s
     end
+    opt.on('--label METRIC_LABEL', "Label to apply to all metrics (default: #{PrometheusExporter::DEFAULT_LABEL})") do |o|
+      options[:label] = JSON.parse(o.to_s)
+    end
     opt.on('-c', '--collector FILE', String, "(optional) Custom collector to run") do |o|
       custom_collector_filename = o.to_s
     end
@@ -81,7 +91,7 @@ def run
 
   runner = PrometheusExporter::Server::Runner.new(options)
 
-  puts "#{Time.now} Starting prometheus exporter on port #{runner.port}"
+  puts "#{Time.now} Starting prometheus exporter on #{runner.bind}:#{runner.port}"
   runner.start
   sleep
 end

data/lib/prometheus_exporter.rb

@@ -7,7 +7,9 @@ require "thread"
 module PrometheusExporter
   # per: https://github.com/prometheus/prometheus/wiki/Default-port-allocations
   DEFAULT_PORT = 9394
+  DEFAULT_BIND_ADDRESS = 'localhost'
   DEFAULT_PREFIX = 'ruby_'
+  DEFAULT_LABEL = {}
   DEFAULT_TIMEOUT = 2
 
   class OjCompat
@@ -19,6 +21,17 @@ module PrometheusExporter
     end
   end
 
+  def self.hostname
+    @hostname ||=
+      begin
+        require 'socket'
+        Socket.gethostname
+      rescue => e
+        STDERR.puts "Unable to lookup hostname #{e}"
+        "unknown-host"
+      end
+  end
+
   def self.detect_json_serializer(preferred)
     if preferred.nil?
       preferred = :oj if has_oj?

data/lib/prometheus_exporter/client.rb

@@ -53,7 +53,14 @@ module PrometheusExporter
     MAX_SOCKET_AGE = 25
     MAX_QUEUE_SIZE = 10_000
 
-    def initialize(host: 'localhost', port: PrometheusExporter::DEFAULT_PORT, max_queue_size: nil, thread_sleep: 0.5, json_serializer: nil, custom_labels: nil)
+    def initialize(
+      host: ENV.fetch('PROMETHEUS_EXPORTER_HOST', 'localhost'),
+      port: ENV.fetch('PROMETHEUS_EXPORTER_PORT', PrometheusExporter::DEFAULT_PORT),
+      max_queue_size: nil,
+      thread_sleep: 0.5,
+      json_serializer: nil,
+      custom_labels: nil
+    )
       @metrics = []
 
       @queue = Queue.new

data/lib/prometheus_exporter/instrumentation.rb

@@ -4,8 +4,10 @@ require_relative "client"
 require_relative "instrumentation/process"
 require_relative "instrumentation/method_profiler"
 require_relative "instrumentation/sidekiq"
+require_relative "instrumentation/sidekiq_queue"
 require_relative "instrumentation/delayed_job"
 require_relative "instrumentation/puma"
 require_relative "instrumentation/hutch"
 require_relative "instrumentation/unicorn"
 require_relative "instrumentation/active_record"
+require_relative "instrumentation/shoryuken"

data/lib/prometheus_exporter/instrumentation/active_record.rb

@@ -51,17 +51,6 @@ module PrometheusExporter::Instrumentation
     def initialize(metric_labels, config_labels)
       @metric_labels = metric_labels
       @config_labels = config_labels
-      @hostname = nil
-    end
-
-    def hostname
-      @hostname ||=
-        begin
-          `hostname`.strip
-        rescue => e
-          STDERR.puts "Unable to lookup hostname #{e}"
-          "unknown-host"
-        end
     end
 
     def collect
@@ -87,7 +76,7 @@ module PrometheusExporter::Instrumentation
         metric = {
           pid: pid,
           type: "active_record",
-          hostname: hostname,
+          hostname: ::PrometheusExporter.hostname,
           metric_labels: labels
         }
         metric.merge!(pool.stat)

data/lib/prometheus_exporter/instrumentation/process.rb

@@ -3,6 +3,8 @@
 # collects stats from currently running process
 module PrometheusExporter::Instrumentation
   class Process
+    @thread = nil if !defined?(@thread)
+
     def self.start(client: nil, type: "ruby", frequency: 30, labels: nil)
 
       metric_labels =
@@ -42,24 +44,13 @@ module PrometheusExporter::Instrumentation
 
     def initialize(metric_labels)
       @metric_labels = metric_labels
-      @hostname = nil
-    end
-
-    def hostname
-      @hostname ||=
-        begin
-          `hostname`.strip
-        rescue => e
-          STDERR.puts "Unable to lookup hostname #{e}"
-          "unknown-host"
-        end
     end
 
     def collect
       metric = {}
       metric[:type] = "process"
       metric[:metric_labels] = @metric_labels
-      metric[:hostname] = hostname
+      metric[:hostname] = ::PrometheusExporter.hostname
       collect_gc_stats(metric)
       collect_v8_stats(metric)
       collect_process_stats(metric)

data/lib/prometheus_exporter/instrumentation/shoryuken.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module PrometheusExporter::Instrumentation
+  class Shoryuken
+
+    def initialize(client: nil)
+      @client = client || PrometheusExporter::Client.default
+    end
+
+    def call(worker, queue, msg, body)
+      success = false
+      start = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      result = yield
+      success = true
+      result
+    rescue ::Shoryuken::Shutdown => e
+      shutdown = true
+      raise e
+    ensure
+      duration = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - start
+      @client.send_json(
+          type: "shoryuken",
+          queue: queue,
+          name: worker.class.name,
+          success: success,
+          shutdown: shutdown,
+          duration: duration
+      )
+    end
+  end
+end

data/lib/prometheus_exporter/instrumentation/sidekiq_queue.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module PrometheusExporter::Instrumentation
+  class SidekiqQueue
+    def self.start(client: nil, frequency: 30)
+      client ||= PrometheusExporter::Client.default
+      sidekiq_queue_collector = new
+
+      Thread.new do
+        loop do
+          begin
+            client.send_json(sidekiq_queue_collector.collect)
+          rescue StandardError => e
+            STDERR.puts("Prometheus Exporter Failed To Collect Sidekiq Queue metrics #{e}")
+          ensure
+            sleep frequency
+          end
+        end
+      end
+    end
+
+    def collect
+      {
+        type: 'sidekiq_queue',
+        queues: collect_queue_stats
+      }
+    end
+
+    def collect_queue_stats
+      ::Sidekiq::Queue.all.map do |queue|
+        {
+          backlog_total: queue.size,
+          latency_seconds: queue.latency.to_i,
+          labels: { queue: queue.name }
+        }
+      end
+    end
+  end
+end

data/lib/prometheus_exporter/metric/base.rb

@@ -2,6 +2,10 @@
 
 module PrometheusExporter::Metric
   class Base
+
+    @default_prefix = nil if !defined?(@default_prefix)
+    @default_labels = nil if !defined?(@default_labels)
+
     # prefix applied to all metrics
     def self.default_prefix=(name)
       @default_prefix = name
@@ -38,6 +42,10 @@ module PrometheusExporter::Metric
       raise "Not implemented"
     end
 
+    def to_h
+      raise "Not implemented"
+    end
+
     def from_json(json)
       json = JSON.parse(json) if String === json
       @name = json["name"]

data/lib/prometheus_exporter/metric/counter.rb

@@ -23,6 +23,14 @@ module PrometheusExporter::Metric
       end.join("\n")
     end
 
+    def to_h
+      @data.dup
+    end
+
+    def remove(labels)
+      @data.delete(labels)
+    end
+
     def observe(increment = 1, labels = {})
       @data[labels] ||= 0
       @data[labels] += increment

data/lib/prometheus_exporter/metric/gauge.rb

@@ -23,6 +23,14 @@ module PrometheusExporter::Metric
       @data = {}
     end
 
+    def to_h
+      @data.dup
+    end
+
+    def remove(labels)
+      @data.delete(labels)
+    end
+
     def observe(value, labels = {})
       if value.nil?
         data.delete(labels)

data/lib/prometheus_exporter/metric/histogram.rb

@@ -17,6 +17,22 @@ module PrometheusExporter::Metric
       @observations = {}
     end
 
+    def to_h
+      data = {}
+      @observations.each do |labels, buckets|
+        count = @counts[labels]
+        sum = @sums[labels]
+        data[labels] = { "count" => count, "sum" => sum }
+      end
+      data
+    end
+
+    def remove(labels)
+      @observations.delete(labels)
+      @counts.delete(labels)
+      @sums.delete(labels)
+    end
+
     def type
       "histogram"
     end

data/lib/prometheus_exporter/metric/summary.rb

@@ -22,6 +22,23 @@ module PrometheusExporter::Metric
       @sums = {}
     end
 
+    def to_h
+      data = {}
+      calculate_all_quantiles.each do |labels, quantiles|
+        count = @counts[labels]
+        sum = @sums[labels]
+        data[labels] = { "count" => count, "sum" => sum }
+      end
+      data
+    end
+
+    def remove(labels)
+      @counts.delete(labels)
+      @sums.delete(labels)
+      @buffers[0].delete(labels)
+      @buffers[1].delete(labels)
+    end
+
     def type
       "summary"
     end

data/lib/prometheus_exporter/server.rb

@@ -5,6 +5,7 @@ require_relative "server/type_collector"
 require_relative "server/web_collector"
 require_relative "server/process_collector"
 require_relative "server/sidekiq_collector"
+require_relative "server/sidekiq_queue_collector"
 require_relative "server/delayed_job_collector"
 require_relative "server/collector_base"
 require_relative "server/collector"
@@ -14,3 +15,4 @@ require_relative "server/puma_collector"
 require_relative "server/hutch_collector"
 require_relative "server/unicorn_collector"
 require_relative "server/active_record_collector"
+require_relative "server/shoryuken_collector"

data/lib/prometheus_exporter/server/active_record_collector.rb

@@ -27,6 +27,7 @@ module PrometheusExporter::Server
 
       @active_record_metrics.map do |m|
         metric_key = (m["metric_labels"] || {}).merge("pid" => m["pid"])
+        metric_key.merge!(m["custom_labels"]) if m["custom_labels"]
 
         ACTIVE_RECORD_GAUGES.map do |k, help|
           k = k.to_s

data/lib/prometheus_exporter/server/collector.rb

@@ -13,11 +13,13 @@ module PrometheusExporter::Server
       register_collector(WebCollector.new)
       register_collector(ProcessCollector.new)
       register_collector(SidekiqCollector.new)
+      register_collector(SidekiqQueueCollector.new)
       register_collector(DelayedJobCollector.new)
       register_collector(PumaCollector.new)
       register_collector(HutchCollector.new)
       register_collector(UnicornCollector.new)
       register_collector(ActiveRecordCollector.new)
+      register_collector(ShoryukenCollector.new)
     end
 
     def register_collector(collector)

data/lib/prometheus_exporter/server/delayed_job_collector.rb

@@ -2,6 +2,17 @@
 
 module PrometheusExporter::Server
   class DelayedJobCollector < TypeCollector
+    def initialize
+      @delayed_jobs_total = nil
+      @delayed_job_duration_seconds = nil
+      @delayed_jobs_total = nil
+      @delayed_failed_jobs_total = nil
+      @delayed_jobs_max_attempts_reached_total = nil
+      @delayed_job_duration_seconds_summary = nil
+      @delayed_job_attempts_summary = nil
+      @delayed_jobs_enqueued = nil
+      @delayed_jobs_pending = nil
+    end
 
     def type
       "delayed_job"

data/lib/prometheus_exporter/server/hutch_collector.rb

@@ -2,6 +2,12 @@
 
 module PrometheusExporter::Server
   class HutchCollector < TypeCollector
+    def initialize
+      @hutch_jobs_total = nil
+      @hutch_job_duration_seconds = nil
+      @hutch_jobs_total = nil
+      @hutch_failed_jobs_total = nil
+    end
 
     def type
       "hutch"

data/lib/prometheus_exporter/server/runner.rb

@@ -9,6 +9,13 @@ module PrometheusExporter::Server
 
   class Runner
     def initialize(options = {})
+      @timeout = nil
+      @port = nil
+      @bind = nil
+      @collector_class = nil
+      @type_collectors = nil
+      @prefix = nil
+
       options.each do |k, v|
         send("#{k}=", v) if self.class.method_defined?("#{k}=")
       end
@@ -16,6 +23,7 @@ module PrometheusExporter::Server
 
     def start
       PrometheusExporter::Metric::Base.default_prefix = prefix
+      PrometheusExporter::Metric::Base.default_labels = label
 
       register_type_collectors
 
@@ -32,73 +40,54 @@ module PrometheusExporter::Server
         )
       end
 
-      server = server_class.new port: port, collector: collector, timeout: timeout, verbose: verbose
+      server = server_class.new port: port, bind: bind, collector: collector, timeout: timeout, verbose: verbose
       server.start
     end
 
-    def prefix=(prefix)
-      @prefix = prefix
-    end
+    attr_accessor :unicorn_listen_address, :unicorn_pid_file
+    attr_writer :prefix, :port, :bind, :collector_class, :type_collectors, :timeout, :verbose, :server_class, :label
 
     def prefix
       @prefix || PrometheusExporter::DEFAULT_PREFIX
     end
 
-    def port=(port)
-      @port = port
-    end
-
     def port
       @port || PrometheusExporter::DEFAULT_PORT
     end
 
-    def collector_class=(collector_class)
-      @collector_class = collector_class
+    def bind
+      @bind || PrometheusExporter::DEFAULT_BIND_ADDRESS
     end
 
     def collector_class
       @collector_class || PrometheusExporter::Server::Collector
     end
 
-    def type_collectors=(type_collectors)
-      @type_collectors = type_collectors
-    end
-
     def type_collectors
       @type_collectors || []
     end
 
-    def timeout=(timeout)
-      @timeout = timeout
-    end
-
     def timeout
       @timeout || PrometheusExporter::DEFAULT_TIMEOUT
     end
 
-    def verbose=(verbose)
-      @verbose = verbose
-    end
-
     def verbose
       return @verbose if defined? @verbose
       false
     end
 
-    def server_class=(server_class)
-      @server_class = server_class
-    end
-
     def server_class
       @server_class || PrometheusExporter::Server::WebServer
     end
 
-    attr_accessor :unicorn_listen_address, :unicorn_pid_file
-
     def collector
       @_collector ||= collector_class.new
     end
 
+    def label
+      @label ||= PrometheusExporter::DEFAULT_LABEL
+    end
+
     private
 
     def register_type_collectors

data/lib/prometheus_exporter/server/shoryuken_collector.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module PrometheusExporter::Server
+  class ShoryukenCollector < TypeCollector
+
+    def initialize
+      @shoryuken_jobs_total = nil
+      @shoryuken_job_duration_seconds = nil
+      @shoryuken_jobs_total = nil
+      @shoryuken_restarted_jobs_total = nil
+      @shoryuken_failed_jobs_total = nil
+    end
+
+    def type
+      "shoryuken"
+    end
+
+    def collect(obj)
+      default_labels = { job_name: obj['name'] , queue_name: obj['queue'] }
+      custom_labels = obj['custom_labels']
+      labels = custom_labels.nil? ? default_labels : default_labels.merge(custom_labels)
+
+      ensure_shoryuken_metrics
+      @shoryuken_job_duration_seconds.observe(obj["duration"], labels)
+      @shoryuken_jobs_total.observe(1, labels)
+      @shoryuken_restarted_jobs_total.observe(1, labels) if obj["shutdown"]
+      @shoryuken_failed_jobs_total.observe(1, labels) if !obj["success"] && !obj["shutdown"]
+    end
+
+    def metrics
+      if @shoryuken_jobs_total
+        [
+            @shoryuken_job_duration_seconds,
+            @shoryuken_jobs_total,
+            @shoryuken_restarted_jobs_total,
+            @shoryuken_failed_jobs_total,
+        ]
+      else
+        []
+      end
+    end
+
+    protected
+
+    def ensure_shoryuken_metrics
+      if !@shoryuken_jobs_total
+
+        @shoryuken_job_duration_seconds =
+            PrometheusExporter::Metric::Counter.new(
+                "shoryuken_job_duration_seconds", "Total time spent in shoryuken jobs.")
+
+        @shoryuken_jobs_total =
+            PrometheusExporter::Metric::Counter.new(
+                "shoryuken_jobs_total", "Total number of shoryuken jobs executed.")
+
+        @shoryuken_restarted_jobs_total =
+            PrometheusExporter::Metric::Counter.new(
+                "shoryuken_restarted_jobs_total", "Total number of shoryuken jobs that we restarted because of a shoryuken shutdown.")
+
+        @shoryuken_failed_jobs_total =
+            PrometheusExporter::Metric::Counter.new(
+                "shoryuken_failed_jobs_total", "Total number of failed shoryuken jobs.")
+
+      end
+    end
+  end
+end

data/lib/prometheus_exporter/server/sidekiq_collector.rb

@@ -3,6 +3,15 @@
 module PrometheusExporter::Server
   class SidekiqCollector < TypeCollector
 
+    def initialize
+      @sidekiq_jobs_total = nil
+      @sidekiq_job_duration_seconds = nil
+      @sidekiq_jobs_total = nil
+      @sidekiq_restarted_jobs_total = nil
+      @sidekiq_failed_jobs_total = nil
+      @sidekiq_dead_jobs_total = nil
+    end
+
     def type
       "sidekiq"
     end

data/lib/prometheus_exporter/server/sidekiq_queue_collector.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+module PrometheusExporter::Server
+  class SidekiqQueueCollector < TypeCollector
+    MAX_SIDEKIQ_METRIC_AGE = 60
+
+    SIDEKIQ_QUEUE_GAUGES = {
+      'backlog_total' => 'Size of the sidekiq queue.',
+      'latency_seconds' => 'Latency of the sidekiq queue.',
+    }.freeze
+
+    attr_reader :sidekiq_metrics, :gauges
+
+    def initialize
+      @sidekiq_metrics = []
+      @gauges = {}
+    end
+
+    def type
+      'sidekiq_queue'
+    end
+
+    def metrics
+      sidekiq_metrics.map do |metric|
+        labels = metric.fetch("labels", {})
+        SIDEKIQ_QUEUE_GAUGES.map do |name, help|
+          if (value = metric[name])
+            gauge = gauges[name] ||= PrometheusExporter::Metric::Gauge.new("sidekiq_queue_#{name}", help)
+            gauge.observe(value, labels)
+          end
+        end
+      end
+
+      gauges.values
+    end
+
+    def collect(object)
+      now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      object['queues'].each do |queue|
+        queue["created_at"] = now
+        queue["labels"].merge!(object['custom_labels']) if object['custom_labels']
+        sidekiq_metrics.delete_if { |metric| metric['created_at'] + MAX_SIDEKIQ_METRIC_AGE < now }
+        sidekiq_metrics << queue
+      end
+    end
+  end
+end

data/lib/prometheus_exporter/server/web_collector.rb

@@ -4,6 +4,11 @@ module PrometheusExporter::Server
   class WebCollector < TypeCollector
     def initialize
       @metrics = {}
+      @http_requests_total = nil
+      @http_duration_seconds = nil
+      @http_redis_duration_seconds = nil
+      @http_sql_duration_seconds = nil
+      @http_queue_duration_seconds = nil
     end
 
     def type

data/lib/prometheus_exporter/server/web_server.rb

@@ -9,7 +9,7 @@ module PrometheusExporter::Server
   class WebServer
     attr_reader :collector
 
-    def initialize(port: , collector: nil, timeout: PrometheusExporter::DEFAULT_TIMEOUT, verbose: false)
+    def initialize(port: , bind: nil, collector: nil, timeout: PrometheusExporter::DEFAULT_TIMEOUT, verbose: false)
 
       @verbose = verbose
 
@@ -38,6 +38,7 @@ module PrometheusExporter::Server
 
       @server = WEBrick::HTTPServer.new(
         Port: port,
+        BindAddress: bind,
         Logger: logger,
         AccessLog: access_log,
       )

data/lib/prometheus_exporter/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PrometheusExporter
-  VERSION = "0.4.16"
+  VERSION = '0.5.3'
 end

data/prometheus_exporter.gemspec

@@ -5,26 +5,28 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require "prometheus_exporter/version"
 
 Gem::Specification.new do |spec|
-  spec.name          = "prometheus_exporter"
-  spec.version       = PrometheusExporter::VERSION
-  spec.authors       = ["Sam Saffron"]
-  spec.email         = ["sam.saffron@gmail.com"]
+  spec.name                 = "prometheus_exporter"
+  spec.version              = PrometheusExporter::VERSION
+  spec.authors              = ["Sam Saffron"]
+  spec.email                = ["sam.saffron@gmail.com"]
 
-  spec.summary       = %q{Prometheus Exporter}
-  spec.description   = %q{Prometheus metric collector and exporter for Ruby}
-  spec.homepage      = "https://github.com/discourse/prometheus_exporter"
-  spec.license       = "MIT"
+  spec.summary              = %q{Prometheus Exporter}
+  spec.description          = %q{Prometheus metric collector and exporter for Ruby}
+  spec.homepage             = "https://github.com/discourse/prometheus_exporter"
+  spec.license              = "MIT"
 
-  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+  spec.post_install_message = "prometheus_exporter will only bind to localhost by default as of v0.5"
+
+  spec.files = `git ls-files -z`.split("\x0").reject do |f|
     f.match(%r{^(test|spec|features|bin)/})
   end
-  spec.bindir        = "bin"
-  spec.executables   = ["prometheus_exporter"]
-  spec.require_paths = ["lib"]
+  spec.bindir               = "bin"
+  spec.executables          = ["prometheus_exporter"]
+  spec.require_paths        = ["lib"]
 
   spec.add_development_dependency "rubocop", ">= 0.69"
   spec.add_development_dependency "bundler", "> 1.16"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "minitest", "~> 5.0"
   spec.add_development_dependency "guard", "~> 2.0"
   spec.add_development_dependency "mini_racer", "~> 0.1"
@@ -32,6 +34,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "oj", "~> 3.0"
   spec.add_development_dependency "rack-test", "~> 0.8.3"
   spec.add_development_dependency "minitest-stub-const", "~> 0.6"
+  spec.add_development_dependency "rubocop-discourse", ">2"
   if !RUBY_ENGINE == 'jruby'
     spec.add_development_dependency "raindrops", "~> 0.19"
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: prometheus_exporter
 version: !ruby/object:Gem::Version
-  version: 0.4.16
+  version: 0.5.3
 platform: ruby
 authors:
 - Sam Saffron
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-11-03 00:00:00.000000000 Z
+date: 2020-07-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rubocop
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -150,6 +150,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: rubocop-discourse
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: '2'
 description: Prometheus metric collector and exporter for Ruby
 email:
 - sam.saffron@gmail.com
@@ -180,7 +194,9 @@ files:
 - lib/prometheus_exporter/instrumentation/method_profiler.rb
 - lib/prometheus_exporter/instrumentation/process.rb
 - lib/prometheus_exporter/instrumentation/puma.rb
+- lib/prometheus_exporter/instrumentation/shoryuken.rb
 - lib/prometheus_exporter/instrumentation/sidekiq.rb
+- lib/prometheus_exporter/instrumentation/sidekiq_queue.rb
 - lib/prometheus_exporter/instrumentation/unicorn.rb
 - lib/prometheus_exporter/metric.rb
 - lib/prometheus_exporter/metric/base.rb
@@ -198,7 +214,9 @@ files:
 - lib/prometheus_exporter/server/process_collector.rb
 - lib/prometheus_exporter/server/puma_collector.rb
 - lib/prometheus_exporter/server/runner.rb
+- lib/prometheus_exporter/server/shoryuken_collector.rb
 - lib/prometheus_exporter/server/sidekiq_collector.rb
+- lib/prometheus_exporter/server/sidekiq_queue_collector.rb
 - lib/prometheus_exporter/server/type_collector.rb
 - lib/prometheus_exporter/server/unicorn_collector.rb
 - lib/prometheus_exporter/server/web_collector.rb
@@ -209,7 +227,8 @@ homepage: https://github.com/discourse/prometheus_exporter
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message: prometheus_exporter will only bind to localhost by default as
+  of v0.5
 rdoc_options: []
 require_paths:
 - lib
@@ -225,7 +244,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.0.3
-signing_key: 
+signing_key:
 specification_version: 4
 summary: Prometheus Exporter
 test_files: []