prometheus_exporter 0.5.1 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4855e62023cc91a24d7f604765a4c34c3cc652421e2f60d6041b8b87376dd74f
-  data.tar.gz: 9772f78ab97d6ae4218afefc353660f1c162df5c801aca60f326e0eeda1894c9
+  metadata.gz: 562647915cd81e672056a83fc79a031d2a0ebb2b4a7a5e32f5786ae5c7d480a7
+  data.tar.gz: 525e28d0cbe4e853c91bb463970fd231b134dba6b5a190c61a2ff7710074fb7b
 SHA512:
-  metadata.gz: 2e1450823b8b770a2538891a4315451e5be4dbe6bc3ffe77ba0b0081c6f436b5a2cfe189c4e4c77442bad72180f0ba6144c074378b55c266d5e6c33b4aa40f42
-  data.tar.gz: 62457c9bdfaedfc02acbdfd0e472790a4ba5b5c124778f8f58173fc0b98f442d7d23e71fd16345334fe71778e46096559e7c2ae32c94255f155abb0dc3a92619
+  metadata.gz: 324b3b0acd1c97e8b5535dff8f57500b4dc3f2433456372c609accbc78d1baf839d82a1e38eabb543f043bbc9fefb401ab1460ba252ee8b9330fe0a91acffc1d
+  data.tar.gz: b2a0082d4e2431cc1f24f11a6af86d6965e2db7c93df1021ed58b38ff8651c84d1e8939b42a13b1b7cac60c9237435ff838a24918ed0ee18d13c3df2aece76d4

data/.github/workflows/ci.yml

@@ -0,0 +1,42 @@
+name: Test Exporter
+
+on:
+  push:
+  pull_request:
+  schedule:
+    - cron: '0 0 * * 0' # weekly
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    name: Ruby ${{ matrix.ruby }}
+    strategy:
+      matrix:
+        ruby: ["2.7", "2.6", "2.5"]
+    steps:
+      - uses: actions/checkout@master
+        with:
+          fetch-depth: 1
+      - uses: actions/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+      - uses: actions/cache@v2
+        with:
+          path: vendor/bundle
+          key: ${{ runner.os }}-${{ matrix.ruby }}-gems-v2-${{ hashFiles('**/Gemfile.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.ruby }}-gems-v2-
+      - name: Setup gems
+        run: |
+          gem install bundler
+          # for Ruby <= 2.6 , details https://github.com/rubygems/rubygems/issues/3284
+          gem update --system 3.0.8 && gem update --system
+          bundle config path vendor/bundle
+          bundle install --jobs 4
+          bundle exec appraisal install
+      - name: Rubocop
+        run: bundle exec rubocop
+      - name: install gems
+        run: bundle exec appraisal bundle
+      - name: Run tests
+        run: bundle exec appraisal rake

data/.gitignore

@@ -7,5 +7,7 @@
 /spec/reports/
 /tmp/
 Gemfile.lock
+/gemfiles/*.gemfile.lock
+
 
 .rubocop-https---raw-githubusercontent-com-discourse-discourse-master--rubocop-yml

data/.rubocop.yml

@@ -1 +1,7 @@
-inherit_from: https://raw.githubusercontent.com/discourse/discourse/master/.rubocop.yml
+inherit_gem:
+  rubocop-discourse: default.yml
+
+AllCops:
+  Exclude:
+    - 'gemfiles/**/*'
+    - 'vendor/**/*'

data/Appraisals

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+appraise "ar-60" do
+  # we are using this version as default in gemspec
+  # gem "activerecord", "~> 6.0.0"
+end
+
+appraise "ar-61" do
+  gem "activerecord", "~> 6.1.1"
+end

data/CHANGELOG

@@ -1,13 +1,46 @@
-0.5.1 - 25-02-2019
+0.8.0 - 05-07-2021
+
+- FIX: handle ThreadError more gracefully in cases where process shuts down
+- FEATURE: add job_name and queue_name labels to delayed job metrics
+- FEATURE: always scope puma metrics on hostname in collector
+- FEATURE: add customizable labels option to puma collector
+- FEATURE: support for Rescue
+- DEV: Remove support for EOL ruby 2.5
+- FIX: Add source location to MethodProfiler patches
+- FEATURE: Improve Active Record instrumentation
+- FEATURE: Support HTTP_X_AMZN_TRACE_ID when supplied
+
+0.7.0 - 29-12-2020
+
+- Dev: Removed support from EOL rubies, only 2.5, 2.6, 2.7 and 3.0 are supported now.
+- Dev: Better support for Ruby 3.0, explicitly depending on webrick
+- Dev: Rails 6.1 instrumentation support
+- FEATURE: clean pattern for overriding middleware labels was introduced (in README)
+- Fix: Better support for forking
+
+0.6.0 - 17-11-2020
+
+- FEATURE: add support for basic-auth in the prometheus_exporter web server
+
+0.5.3 - 29-07-2020
+
+- FEATURE: added #remove to all metric types so users can remove specific labels if needed
+
+0.5.2 - 01-07-2020
+
+- FEATURE: expanded instrumentation for sidekiq
+- FEATURE: configurable default labels
+
+0.5.1 - 25-02-2020
 
 - FEATURE: Allow configuring the default client's host and port via environment variables
 
-0.5.0 - 14-02-2019
+0.5.0 - 14-02-2020
 
 - Breaking change: listen only to localhost by default to prevent unintended insecure configuration
 - FIX: Avoid calling `hostname` aggressively, instead cache it on the exporter instance
 
-0.4.17 - 13-01-2019
+0.4.17 - 13-01-2020
 
 - FEATURE: add support for `to_h` on all metrics which can be used to query existing key/values
 

data/README.md

@@ -19,6 +19,7 @@ To learn more see [Instrumenting Rails with Prometheus](https://samsaffron.com/a
     * [Hutch metrics](#hutch-message-processing-tracer)
   * [Puma metrics](#puma-metrics)
   * [Unicorn metrics](#unicorn-process-metrics)
+  * [Resque metrics](#resque-metrics)
   * [Custom type collectors](#custom-type-collectors)
   * [Multi process mode with custom collector](#multi-process-mode-with-custom-collector)
   * [GraphQL support](#graphql-support)
@@ -33,7 +34,7 @@ To learn more see [Instrumenting Rails with Prometheus](https://samsaffron.com/a
 
 ## Requirements
 
-Minimum Ruby of version 2.3.0 is required, Ruby 2.2.0 is EOL as of 2018-03-31
+Minimum Ruby of version 2.5.0 is required, Ruby 2.4.0 is EOL as of 2020-04-05
 
 ## Installation
 
@@ -190,6 +191,64 @@ Ensure you run the exporter in a monitored background process:
 $ bundle exec prometheus_exporter
 ```
 
+#### Metrics collected by Rails integration middleware
+
+| Type    | Name                            | Description                                                 |
+| ---     | ---                             | ---                                                         |
+| Counter | `http_requests_total`           | Total HTTP requests from web app                            |
+| Summary | `http_duration_seconds`         | Time spent in HTTP reqs in seconds                          |
+| Summary | `http_redis_duration_seconds`¹  | Time spent in HTTP reqs in Redis, in seconds                |
+| Summary | `http_sql_duration_seconds`²    | Time spent in HTTP reqs in SQL in seconds                   |
+| Summary | `http_queue_duration_seconds`³  | Time spent queueing the request in load balancer in seconds |
+
+All metrics have a `controller` and an `action` label.
+`http_requests_total` additionally has a (HTTP response) `status` label.
+
+To add your own labels to the default metrics, create a subclass of `PrometheusExporter::Middleware`, override `custom_labels`, and use it in your initializer.
+```ruby
+class MyMiddleware < PrometheusExporter::Middleware
+  def custom_labels(env)
+    labels = {}
+
+    if env['HTTP_X_PLATFORM']
+      labels['platform'] = env['HTTP_X_PLATFORM']
+    end
+
+    labels
+  end
+end
+```
+
+If you're not using Rails like framework, you can extend `PrometheusExporter::Middleware#default_labels` in a way to add more relevant labels.
+For example you can mimic [prometheus-client](https://github.com/prometheus/client_ruby) labels with code like this:
+```ruby
+class MyMiddleware < PrometheusExporter::Middleware
+  def default_labels(env, result)
+    status = (result && result[0]) || -1
+    path = [env["SCRIPT_NAME"], env["PATH_INFO"]].join
+    {
+      path: strip_ids_from_path(path),
+      method: env["REQUEST_METHOD"],
+      status: status
+    }
+  end
+
+  def strip_ids_from_path(path)
+    path
+      .gsub(%r{/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}(/|$)}, '/:uuid\\1')
+      .gsub(%r{/\d+(/|$)}, '/:id\\1')
+  end
+end
+```
+That way you won't have all metrics labeled with `controller=other` and `action=other`, but have labels such as
+```
+ruby_http_duration_seconds{path="/api/v1/teams/:id",method="GET",status="200",quantile="0.99"} 0.009880661998977303
+```
+
+¹) Only available when Redis is used.
+²) Only available when Mysql or PostgreSQL are used.
+³) Only available when [Instrumenting Request Queueing Time](#instrumenting-request-queueing-time) is set up.
+
 #### Activerecord Connection Pool Metrics
 
 This collects activerecord connection pool metrics.
@@ -244,6 +303,19 @@ Sidekiq.configure_server do |config|
 end
 ```
 
+##### Metrics collected by ActiveRecord Instrumentation
+
+| Type  | Name                                        | Description                           |
+| ---   | ---                                         | ---                                   |
+| Gauge | `active_record_connection_pool_connections` | Total connections in pool             |
+| Gauge | `active_record_connection_pool_busy`        | Connections in use in pool            |
+| Gauge | `active_record_connection_pool_dead`        | Dead connections in pool              |
+| Gauge | `active_record_connection_pool_idle`        | Idle connections in pool              |
+| Gauge | `active_record_connection_pool_waiting`     | Connection requests waiting           |
+| Gauge | `active_record_connection_pool_size`        | Maximum allowed connection pool size  |
+
+All metrics collected by the ActiveRecord integration include at least the following labels: `pid` (of the process the stats where collected in), `pool_name`, any labels included in the `config_labels` option (prefixed with `dbconfig_`, example: `dbconfig_host`), and all custom labels provided with the `custom_labels` option.
+
 #### Per-process stats
 
 You may also be interested in per-process stats. This collects memory and GC stats:
@@ -260,11 +332,30 @@ end
 # in unicorn/puma/passenger be sure to run a new process instrumenter after fork
 after_fork do
   require 'prometheus_exporter/instrumentation'
-  PrometheusExporter::Instrumentation::Process.start(type:"web")
+  PrometheusExporter::Instrumentation::Process.start(type: "web")
 end
 
 ```
 
+##### Metrics collected by Process Instrumentation
+
+| Type    | Name                      | Description                                  |
+| ---     | ---                       | ---                                          |
+| Gauge   | `heap_free_slots`         | Free ruby heap slots                         |
+| Gauge   | `heap_live_slots`         | Used ruby heap slots                         |
+| Gauge   | `v8_heap_size`*           | Total JavaScript V8 heap size (bytes)        |
+| Gauge   | `v8_used_heap_size`*      | Total used JavaScript V8 heap size (bytes)   |
+| Gauge   | `v8_physical_size`*       | Physical size consumed by V8 heaps           |
+| Gauge   | `v8_heap_count`*          | Number of V8 contexts running                |
+| Gauge   | `rss`                     | Total RSS used by process                    |
+| Counter | `major_gc_ops_total`      | Major GC operations by process               |
+| Counter | `minor_gc_ops_total`      | Minor GC operations by process               |
+| Counter | `allocated_objects_total` | Total number of allocated objects by process |
+
+_Metrics marked with * are only collected when `MiniRacer` is defined._
+
+Metrics collected by Process instrumentation include labels `type` (as given with the `type` option), `pid` (of the process the stats where collected in), and any custom labels given to `Process.start` with the `labels` option.
+
 #### Sidekiq metrics
 
 Including Sidekiq metrics (how many jobs ran? how many failed? how long did they take? how many are dead? how many were restarted?)
@@ -279,6 +370,17 @@ Sidekiq.configure_server do |config|
 end
 ```
 
+To monitor Queue size and latency:
+
+```ruby
+Sidekiq.configure_server do |config|
+  config.on :startup do
+    require 'prometheus_exporter/instrumentation'
+    PrometheusExporter::Instrumentation::SidekiqQueue.start
+  end
+end
+```
+
 To monitor Sidekiq process info:
 
 ```ruby
@@ -300,6 +402,35 @@ Sometimes the Sidekiq server shuts down before it can send metrics, that were ge
   end
 ```
 
+##### Metrics collected by Sidekiq Instrumentation
+
+**PrometheusExporter::Instrumentation::Sidekiq**
+| Type    | Name                           | Description                                                                  |
+| ---     | ---                            | ---                                                                          |
+| Summary | `sidekiq_job_duration_seconds` | Time spent in sidekiq jobs                                                   |
+| Counter | `sidekiq_jobs_total`           | Total number of sidekiq jobs executed                                        |
+| Counter | `sidekiq_restarted_jobs_total` | Total number of sidekiq jobs that we restarted because of a sidekiq shutdown |
+| Counter | `sidekiq_failed_jobs_total`    | Total number of failed sidekiq jobs                                          |
+
+All metrics have a `job_name` label and a `queue` label.
+
+**PrometheusExporter::Instrumentation::Sidekiq.death_handler**
+| Type    | Name                      | Description                       |
+| ---     | ---                       | ---                               |
+| Counter | `sidekiq_dead_jobs_total` | Total number of dead sidekiq jobs |
+
+This metric has a `job_name` label and a `queue` label.
+
+**PrometheusExporter::Instrumentation::SidekiqQueue**
+| Type  | Name                            | Description                  |
+| ---   | ---                             | ---                          |
+| Gauge | `sidekiq_queue_backlog_total`   | Size of the sidekiq queue    |
+| Gauge | `sidekiq_queue_latency_seconds` | Latency of the sidekiq queue |
+
+Both metrics will have a `queue` label with the name of the queue.
+
+_See [Metrics collected by Process Instrumentation](#metrics-collected-by-process-instrumentation) for a list of metrics the Process instrumentation will produce._
+
 #### Shoryuken metrics
 
 For Shoryuken metrics (how many jobs ran? how many failed? how long did they take? how many were restarted?)
@@ -313,6 +444,17 @@ Shoryuken.configure_server do |config|
 end
 ```
 
+##### Metrics collected by Shoryuken Instrumentation
+
+| Type    | Name                             | Description                                                                      |
+| ---     | ---                              | ---                                                                              |
+| Counter | `shoryuken_job_duration_seconds` | Total time spent in shoryuken jobs                                               |
+| Counter | `shoryuken_jobs_total`           | Total number of shoryuken jobs executed                                          |
+| Counter | `shoryuken_restarted_jobs_total` | Total number of shoryuken jobs that we restarted because of a shoryuken shutdown |
+| Counter | `shoryuken_failed_jobs_total`    | Total number of failed shoryuken jobs                                            |
+
+All metrics have labels for `job_name` and `queue_name`.
+
 #### Delayed Job plugin
 
 In an initializer:
@@ -324,6 +466,21 @@ unless Rails.env == "test"
 end
 ```
 
+##### Metrics collected by Delayed Job Instrumentation
+
+| Type    | Name                                      | Description                                                        | Labels     |
+| ---     | ---                                       | ---                                                                | ---        |
+| Counter | `delayed_job_duration_seconds`            | Total time spent in delayed jobs                                   | `job_name` |
+| Counter | `delayed_jobs_total`                      | Total number of delayed jobs executed                              | `job_name` |
+| Gauge   | `delayed_jobs_enqueued`                   | Number of enqueued delayed jobs                                    | -          |
+| Gauge   | `delayed_jobs_pending`                    | Number of pending delayed jobs                                     | -          |
+| Counter | `delayed_failed_jobs_total`               | Total number failed delayed jobs executed                          | `job_name` |
+| Counter | `delayed_jobs_max_attempts_reached_total` | Total number of delayed jobs that reached max attempts             | -          |
+| Summary | `delayed_job_duration_seconds_summary`    | Summary of the time it takes jobs to execute                       | `status`   |
+| Summary | `delayed_job_attempts_summary`            | Summary of the amount of attempts it takes delayed jobs to succeed | -          |
+
+All metrics have labels for `job_name` and `queue_name`.
+
 #### Hutch Message Processing Tracer
 
 Capture [Hutch](https://github.com/gocardless/hutch) metrics (how many jobs ran? how many failed? how long did they take?)
@@ -335,13 +492,23 @@ unless Rails.env == "test"
 end
 ```
 
+##### Metrics collected by Hutch Instrumentation
+
+| Type    | Name                         | Description                             |
+| ---     | ---                          | ---                                     |
+| Counter | `hutch_job_duration_seconds` | Total time spent in hutch jobs          |
+| Counter | `hutch_jobs_total`           | Total number of hutch jobs executed     |
+| Counter | `hutch_failed_jobs_total`    | Total number failed hutch jobs executed |
+
+All metrics have a `job_name` label.
+
 #### Instrumenting Request Queueing Time
 
 Request Queueing is defined as the time it takes for a request to reach your application (instrumented by this `prometheus_exporter`) from farther upstream (as your load balancer). A high queueing time usually means that your backend cannot handle all the incoming requests in time, so they queue up (= you should see if you need to add more capacity).
 
 As this metric starts before `prometheus_exporter` can handle the request, you must add a specific HTTP header as early in your infrastructure as possible (we recommend your load balancer or reverse proxy).
 
-Configure your HTTP server / load balancer to add a header `X-Request-Start: t=<MSEC>` when passing the request upstream. For more information, please consult your software manual.
+The Amazon Application Load Balancer [request tracing header](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-request-tracing.html) is natively supported. If you are using another upstream entrypoint, you may configure your HTTP server / load balancer to add a header `X-Request-Start: t=<MSEC>` when passing the request upstream. For more information, please consult your software manual.
 
 Hint: we aim to be API-compatible with the big APM solutions, so if you've got requests queueing time configured for them, it should be expected to also work with `prometheus_exporter`.
 
@@ -359,6 +526,42 @@ after_worker_boot do
 end
 ```
 
+#### Metrics collected by Puma Instrumentation
+
+| Type  | Name                              | Description                                                 |
+| ---   | ---                               | ---                                                         |
+| Gauge | `puma_workers_total`              | Number of puma workers                                      |
+| Gauge | `puma_booted_workers_total`       | Number of puma workers booted                               |
+| Gauge | `puma_old_workers_total`          | Number of old puma workers                                  |
+| Gauge | `puma_running_threads_total`      | Number of puma threads currently running                    |
+| Gauge | `puma_request_backlog_total`      | Number of requests waiting to be processed by a puma thread |
+| Gauge | `puma_thread_pool_capacity_total` | Number of puma threads available at current scale           |
+| Gauge | `puma_max_threads_total`          | Number of puma threads at available at max scale            |
+
+All metrics may have a `phase` label and all custom labels provided with the `labels` option.
+
+### Resque metrics
+
+The resque metrics are using the `Resque.info` method, which queries Redis internally. To start monitoring your resque
+installation, you'll need to start the instrumentation:
+
+```ruby
+# e.g. config/initializers/resque.rb
+require 'prometheus_exporter/instrumentation'
+PrometheusExporter::Instrumentation::Resque.start
+```
+
+#### Metrics collected by Resque Instrumentation
+
+| Type  | Name                   | Description                            |
+| ---   | ---                    | ---                                    |
+| Gauge | `processed_jobs_total` | Total number of processed Resque jobs  |
+| Gauge | `failed_jobs_total`    | Total number of failed Resque jobs     |
+| Gauge | `pending_jobs_total`   | Total number of pending Resque jobs    |
+| Gauge | `queues_total`         | Total number of Resque queues          |
+| Gauge | `workers_total`        | Total number of Resque workers running |
+| Gauge | `working_total`        | Total number of Resque workers working |
+
 ### Unicorn process metrics
 
 In order to gather metrics from unicorn processes, we use `rainbows`, which exposes `Rainbows::Linux.tcp_listener_stats` to gather information about active workers and queued requests. To start monitoring your unicorn processes, you'll need to know both the path to unicorn PID file and the listen address (`pid_file` and `listen` in your unicorn config file)
@@ -374,6 +577,14 @@ prometheus_exporter --unicorn-master /var/run/unicorn.pid --unicorn-listen-addre
 
 Note: You must install the `raindrops` gem in your `Gemfile` or locally.
 
+#### Metrics collected by Unicorn Instrumentation
+
+| Type  | Name                            | Description                                                    |
+| ---   | ---                             | ---                                                            |
+| Gauge | `unicorn_workers_total`         | Number of unicorn workers                                      |
+| Gauge | `unicorn_active_workers_total`  | Number of active unicorn workers                               |
+| Gauge | `unicorn_request_backlog_total` | Number of requests waiting to be processed by a unicorn worker |
+
 ### Custom type collectors
 
 In some cases you may have custom metrics you want to ship the collector in a batch. In this case you may still be interested in the base collector behavior, but would like to add your own special messages.
@@ -432,8 +643,8 @@ Then you can collect the metrics you need on demand:
 
 ```ruby
 def metrics
-  user_count_gague = PrometheusExporter::Metric::Gauge.new('user_count', 'number of users in the app')
-  user_count_gague.observe User.count
+  user_count_gauge = PrometheusExporter::Metric::Gauge.new('user_count', 'number of users in the app')
+  user_count_gauge.observe User.count
   [user_count_gauge]
 end
 ```
@@ -542,6 +753,68 @@ ruby_web_requests{hostname="app-server-01",route="test/route"} 1
 ruby_web_requests{hostname="app-server-01"} 1
 ```
 
+### Exporter Process Configuration
+
+When running the process for `prometheus_exporter` using `bin/prometheus_exporter`, there are several configurations that
+can be passed in:
+
+```
+Usage: prometheus_exporter [options]
+    -p, --port INTEGER               Port exporter should listen on (default: 9394)
+    -b, --bind STRING                IP address exporter should listen on (default: localhost)
+    -t, --timeout INTEGER            Timeout in seconds for metrics endpoint (default: 2)
+        --prefix METRIC_PREFIX       Prefix to apply to all metrics (default: ruby_)
+        --label METRIC_LABEL         Label to apply to all metrics (default: {})
+    -c, --collector FILE             (optional) Custom collector to run
+    -a, --type-collector FILE        (optional) Custom type collectors to run in main collector
+    -v, --verbose
+        --auth FILE                  (optional) enable basic authentication using a htpasswd FILE
+        --realm REALM                (optional) Use REALM for basic authentication (default: "Prometheus Exporter")
+        --unicorn-listen-address ADDRESS
+                                     (optional) Address where unicorn listens on (unix or TCP address)
+        --unicorn-master PID_FILE    (optional) PID file of unicorn master process to monitor unicorn
+```
+
+#### Example
+
+The following will run the process at
+- Port `8080` (default `9394`)
+- Bind to `0.0.0.0` (default `localhost`)
+- Timeout in `1 second` for metrics endpoint (default `2 seconds`)
+- Metric prefix as `foo_` (default `ruby_`)
+- Default labels as `{environment: "integration", foo: "bar"}`
+
+```bash
+prometheus_exporter -p 8080 \
+                    -b 0.0.0.0 \
+                    -t 1 \
+                    --label '{"environment": "integration", "foo": "bar"}' \
+                    --prefix 'foo_'
+```
+
+#### Enabling Basic Authentication
+
+If you desire authentication on your `/metrics` route, you can enable basic authentication with the `--auth` option.
+
+```
+$ prometheus_exporter --auth my-htpasswd-file
+```
+
+Additionally, the `--realm` option may be used to provide a customized realm for the challenge request.
+
+Notes:
+
+* You will need to create a `htpasswd` formatted file before hand which contains one or more user:password entries
+* Only the basic `crypt` encryption is currently supported
+
+A simple `htpasswd` file can be created with the Apache `htpasswd` utility; e.g:
+
+```
+$ htpasswd -cdb my-htpasswd-file my-user my-unencrypted-password
+```
+
+This will create a file named `my-htpasswd-file` which is suitable for use the `--auth` option.
+
 ### Client default labels
 
 You can specify a default label for instrumentation metrics sent by a specific client. For example: