prometheus_exporter 2.2.0 → 2.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e8f8c9307f755aa00e95f323d2be11cdf6830f7e420c9bf70d90260e9bc81a85
-  data.tar.gz: 216bedbba838f0a84391ef2e85b290aa596cf3c42d27b4cc6bce7a9fae8be01d
+  metadata.gz: be510b790b625379993092d05de6b8c7f02e30e74ca0ca630ba5c65ca99385d4
+  data.tar.gz: 43d78c0a9ab134b504cc6e4a6f86d799f9650c5b5e7d7b0d62a33afe3e97374f
 SHA512:
-  metadata.gz: 3ad534e47314779c76eb9e80d5de23619cd0865c2337147188d7f2ddba3bff9a6c9cf630e20720899c4e7ef9c95bf1db317a719d08fc241850f902aceae1b5cf
-  data.tar.gz: 6e71eab8cf6cdb049612c5afc404d6e399a142ee3eb5bbb3ae897e374ac6474d09f1b6c3ad94ab31f8d12fc52fc9e609a8b066051c89e44ea84be0fa0ec2cf0e
+  metadata.gz: d71f41661e4af76837dcb7df1cba50b3a6150ffce3dddebb1dc1710bf9a69dafc6a10750955fda8e870ae62a835e2bbcac417c00256285e3fc4266f3e2194e8d
+  data.tar.gz: 27cf01bfec436b1d6ece387460beefe35d8cf9146ab1d24d27ffebaeeb306df34d7432a31c307aef26e8b59994796593910f90cfaa9982c00a6344a2601a486d

data/CHANGELOG

@@ -1,232 +1,404 @@
-2.2.0 - 2024-12-05
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [2.3.0] - 2025-08-01
+
+- FEATURE: Added puma_busy_threads metric that provides a holistic view of server workload by calculating (active threads - idle threads) + queued requests
+- FEATURE: adding marking_time and sweeping_time to GC metrics
+- DEV: bumped up minimum Ruby version to 3.2
+- FIX: better handling of malformed metrics
+- FIX: get controller labels from controller, not params
+- DEV: expose close on runner instance
+- FEATURE: improved puma metrics
+
+## [2.2.0] - 2024-12-05
+
+### Added
 
-- FIX: Ensure socket is closed when error is raised while opening socket
 - Feature: Add Dalli::Client memcache metrics for web_collector
 
-2.1.1 - 2024-06-19
+### Fixed
+
+- FIX: Ensure socket is closed when error is raised while opening socket
+
+## [2.1.1] - 2024-06-19
+
+### Added
 
 - FEATURE: improve good_job instrumentation
-- FIX: improve Ruby 3.X support
 - FEATURE: imstrumentation for malloc / oldmalloc increace in GC stats
 
-2.1.0 - 2024-01-08
+### Fixed
+
+- FIX: improve Ruby 3.X support
+
+## [2.1.0] - 2024-01-08
+
+### Added
 
 - FEATURE: good_job instrumentation
+
+### Changed
+
 - PERF: improve performance of histogram
 - DEV: use new metric collector pattern so we reuse code between collectors
 
-2.0.8 - 2023-01-20
+## [2.0.8] - 2023-01-20
+
+### Added
 
 - FEATURE: attempting to make our first docker release
 
-2.0.7 - 2023-01-13
+## [2.0.7] - 2023-01-13
 
+### Added
 - FEATURE: allow binding server to both ipv4 and v6
+
+### Fixed
+
 - FIX: expire stale sidekiq metrics
 
 
-2.0.6 - 2022-11-22
+## [2.0.6] - 2022-11-22
+
+### Fixed
 
 - FIX: use user specified labels over default in merge conflict
 - FIX: sidekiq stats collector memory leak
 
-2.0.5 - 2022-11-15
+## [2.0.5] - 2022-11-15
+
+### Fixed
 
 - FIX: regression :prepend style instrumentation not working correctly
 
-2.0.4 - 2022-11-10
+## [2.0.4] - 2022-11-10
+
+### Fixed
 
-- FIX/FEATURE: support for Redis 5 gem instrumentation
+- FIX support for Redis 5 gem instrumentation
 
-2.0.3 - 2022-05-23
+## [2.0.3] - 2022-05-23
+
+### Added
 
 - FEATURE: new ping endpoint for keepalive checks
+
+### Fixed
+
 - FIX: order histogram correctly for GCP support
 - FIX: improve sidekiq instrumentation
 
-2.0.2 - 2022-02-25
+## [2.0.2] - 2022-02-25
+
+### Fixed
 
 - FIX: runner was not requiring unicorn integration correctly leading to a crash
 
-2.0.1 - 2022-02-24
+## [2.0.1] - 2022-02-24
+
+### Fixed
 
 - FIX: ensure threads do not leak when calling #start repeatedly on instrumentation classes, this is an urgent patch for Puma integration
 
-2.0.0 - 2022-02-18
+## [2.0.0] - 2022-02-18
+
+### Added
 
 - FEATURE: Add per worker custom labels
 - FEATURE: support custom histogram buckets
+
+### Fixed
+
 - FIX: all metrics are exposing status label, and not only `http_requests_total`
+
+### Changed
+
 - BREAKING: rename all `http_duration` metrics to `http_request_duration` to match prometheus official naming conventions (See https://prometheus.io/docs/practices/naming/#metric-names).
 
-1.0.1 - 2021-12-22
+## [1.0.1] - 2021-12-22
+
+### Added
 
 - FEATURE: add labels to preflight requests
 - FEATURE: SidekiqStats metrics
+
+### Fixed
+
 - FIX: mintor refactors to Sidekiq metrics
 
-1.0.0 - 2021-11-23
+## [1.0.0] - 2021-11-23
+
+### Added
 
-- BREAKING: rename metrics to match prometheus official naming conventions (See https://prometheus.io/docs/practices/naming/#metric-names)
 - FEATURE: Sidekiq process metrics
 - FEATURE: Allow collecting web metrics as histograms
+
+### Fixed
+
 - FIX: logger improved for web server
 - FIX: Remove job labels from DelayedJob queues
 
-0.8.1 - 2021-08-04
+### Changed
+
+- BREAKING: rename metrics to match prometheus official naming conventions (See https://prometheus.io/docs/practices/naming/#metric-names)
+
+
+## [0.8.1] - 2021-08-04
+
+### Added
 
 - FEATURE: swap from hardcoded STDERR to logger pattern (see README for details)
 
-0.8.0 - 2021-07-05
+## [0.8.0] - 2021-07-05
+
+### Added
 
-- FIX: handle ThreadError more gracefully in cases where process shuts down
 - FEATURE: add job_name and queue_name labels to delayed job metrics
 - FEATURE: always scope puma metrics on hostname in collector
 - FEATURE: add customizable labels option to puma collector
 - FEATURE: support for Resque
-- DEV: Remove support for EOL ruby 2.5
-- FIX: Add source location to MethodProfiler patches
 - FEATURE: Improve Active Record instrumentation
 - FEATURE: Support HTTP_X_AMZN_TRACE_ID when supplied
 
-0.7.0 - 2020-12-29
+### Fixed
+
+- FIX: handle ThreadError more gracefully in cases where process shuts down
+- FIX: Add source location to MethodProfiler patches
+
+### Removed
+
+- DEV: Remove support for EOL ruby 2.5
+
+## [0.7.0] - 2020-12-29
+
+### Added
+
+- FEATURE: clean pattern for overriding middleware labels was introduced (in README)
+
+### Fixed
+
+- Fix: Better support for forking
+
+### Changed
 
 - Dev: Removed support from EOL rubies, only 2.5, 2.6, 2.7 and 3.0 are supported now.
 - Dev: Better support for Ruby 3.0, explicitly depending on webrick
 - Dev: Rails 6.1 instrumentation support
-- FEATURE: clean pattern for overriding middleware labels was introduced (in README)
-- Fix: Better support for forking
 
-0.6.0 - 2020-11-17
+## [0.6.0] - 2020-11-17
+
+### Added
 
 - FEATURE: add support for basic-auth in the prometheus_exporter web server
 
-0.5.3 - 2020-07-29
+## [0.5.3] - 2020-07-29
+
+### Added
 
 - FEATURE: added #remove to all metric types so users can remove specific labels if needed
 
-0.5.2 - 2020-07-01
+## [0.5.2] - 2020-07-01
+
+### Added
 
 - FEATURE: expanded instrumentation for sidekiq
 - FEATURE: configurable default labels
 
-0.5.1 - 2020-02-25
+## [0.5.1] - 2020-02-25
+
+### Added
 
 - FEATURE: Allow configuring the default client's host and port via environment variables
 
-0.5.0 - 2020-02-14
+## [0.5.0] - 2020-02-14
+
+### Fixed
 
-- Breaking change: listen only to localhost by default to prevent unintended insecure configuration
 - FIX: Avoid calling `hostname` aggressively, instead cache it on the exporter instance
 
-0.4.17 - 2020-01-13
+### Changed
+
+- Breaking change: listen only to localhost by default to prevent unintended insecure configuration
+
+## [0.4.17] - 2020-01-13
+
+### Added
 
 - FEATURE: add support for `to_h` on all metrics which can be used to query existing key/values
 
-0.4.16 - 2019-11-04
+## [0.4.16] - 2019-11-04
+
+### Added
 
 - FEATURE: Support #reset! on all metric types to reset a metric to default
 
-0.4.15 - 2019-11-04
+## [0.4.15] - 2019-11-04
+
+### Added
 
 - FEATURE: Improve delayed job collector, add pending counts
 - FEATURE: New ActiveRecord collector (documented in readme)
 - FEATURE: Allow passing in histogram and summary options
 - FEATURE: Allow custom labels for unicorn collector
 
-0.4.14 - 2019-09-10
+## [0.4.14] - 2019-09-10
+
+### Added
 
 - FEATURE: allow finding metrics by name RemoteMetric #find_registered_metric
+
+### Fixed
+
 - FIX: guard socket closing
 
-0.4.13 - 2019-07-09
+## [0.4.13] - 2019-07-09
+
+### Fixed
 
 - Fix: Memory leak in unicorn and puma collectors
 
-0.4.12 - 2019-05-30
+## [0.4.12] - 2019-05-30
+
+### Fixed
 
 - Fix: unicorn collector reporting incorrect number of unicorn workers
 
-0.4.11 - 2019-05-15
+## [0.4.11] - 2019-05-15
+
+### Fixed
 
 - Fix: Handle stopping nil worker_threads in Client
+
+### Changed
+
 - Dev: add frozen string literals
 
-0.4.10 - 2019-04-29
+## [0.4.10] - 2019-04-29
+
+### Fixed
 
 - Fix: Custom label support for puma collector
 - Fix: Raindrops socket collector not working correctly
 
-0.4.9 - 2019-04-11
+## [0.4.9] - 2019-04-11
+
+### Fixed
 
 - Fix: Gem was not working correctly in Ruby 2.4 and below due to a syntax error
 
-0.4.8 - 2019-04-10
+## [0.4.8] - 2019-04-10
+
+### Added
 
 - Feature: added helpers for instrumenting unicorn using raindrops
 
-0.4.7 - 2019-04-08
+## [0.4.7] - 2019-04-08
+
+### Fixed
 
 - Fix: collector was not escaping " \ and \n correctly. This could lead
   to a corrupt payload in some cases.
 
-0.4.6 - 2019-04-02
+## [0.4.6] - 2019-04-02
+
+### Added
 
 - Feature: Allow resetting a counter
 - Feature: Add sidekiq metrics: restarted, dead jobs counters
+
+### Fixed
+
 - Fix: Client shutting down before sending metrics to collector
 
-0.4.5 - 2019-02-14
+## [0.4.5] - 2019-02-14
+
+### Added
 
 - Feature: Allow process collector to ship custom labels for all process metrics
+
+### Fixed
+
 - Fix: Always scope process metrics on hostname in collector
 
-0.4.4 - 2019-02-13
+## [0.4.4] - 2019-02-13
+
+### Added
 
 - Feature: add support for local metric collection without using HTTP
 
-0.4.3 - 2019-02-11
+## [0.4.3] - 2019-02-11
+
+### Added
 
 - Feature: Add alias for Gauge #observe called #set, this makes it a bit easier to migrate from prom
 - Feature: Add increment and decrement to Counter
 
-0.4.2 - 2018-11-30
+## [0.4.2] - 2018-11-30
+
+### Fixed
 
-- Fix/Feature: setting a Gauge to nil will remove Gauge (setting to non numeric will raise)
+- Fix: setting a Gauge to nil will remove Gauge (setting to non numeric will raise)
 
-0.4.0 - 2018-10-23
+## [0.4.0] - 2018-10-23
+
+### Added
 
 - Feature: histogram support
 - Feature: custom quantile support for summary
 - Feature: Puma metrics
+
+### Fixed
+
 - Fix: delayed job metrics
 
-0.3.4 - 2018-10-02
+## [0.3.4] - 2018-10-02
+
+### Fixed
 
 - Fix: custom collector via CLI was not working correctly
 
-0.3.3
+## [0.3.3]
+
+### Added
 
 - Feature: Add more metrics to delayed job collector
 
-0.3.2
+## [0.3.2]
+
+### Added
 
 - Feature: Add posibility to set custom_labels on multi process mode
 
-0.3.1
+## [0.3.1]
+
+### Changed
 
 - Allow runner to accept a --timeout var
 - Allow runner to accept a blank prefix
 
-0.3.0
+## [0.3.0]
+
+### Changed
 
 - Breaking change: Follow Prometheus metric [naming conventions](https://prometheus.io/docs/practices/naming/#metric-names)
 
-0.1.15 - 2018-02-19
+## [0.1.15] - 2018-02-19
+
+### Added
 
 - Feature: Prefer to use oj if it is loadable
 
-0.1.14 - 2018-02-17
+## [0.1.14] - 2018-02-17
+
+### Added
 
-- Fix: error when shipping summary metric with no labels
 - Feature: runner was extracted so it can be reused @304
+
+### Fixed
+
+- Fix: error when shipping summary metric with no labels

data/README.md

@@ -98,8 +98,8 @@ server.collector.register_metric(counter)
 server.collector.register_metric(summary)
 server.collector.register_metric(histogram)
 
-gauge.observe(get_rss)
-gauge.observe(get_rss)
+gauge.observe(server.get_rss)
+gauge.observe(server.get_rss)
 
 counter.observe(1, route: 'test/route')
 counter.observe(1, route: 'another/route')
@@ -373,6 +373,8 @@ end
 | Counter | `major_gc_ops_total`      | Major GC operations by process               |
 | Counter | `minor_gc_ops_total`      | Minor GC operations by process               |
 | Counter | `allocated_objects_total` | Total number of allocated objects by process |
+| Gauge   | `marking_time`            | Marking time spent (Ruby 3.3 minimum)        |
+| Gauge   | `sweeping_time`           | Sweeping time spent (Ruby 3.3 minimum)       |
 
 _Metrics marked with * are only collected when `MiniRacer` is defined._
 
@@ -609,15 +611,16 @@ end
 
 #### Metrics collected by Puma Instrumentation
 
-| Type  | Name                        | Description                                                 |
-| ---   | ---                         | ---                                                         |
-| Gauge | `puma_workers`              | Number of puma workers                                      |
-| Gauge | `puma_booted_workers`       | Number of puma workers booted                               |
-| Gauge | `puma_old_workers`          | Number of old puma workers                                  |
-| Gauge | `puma_running_threads`      | Number of puma threads currently running                    |
-| Gauge | `puma_request_backlog`      | Number of requests waiting to be processed by a puma thread |
-| Gauge | `puma_thread_pool_capacity` | Number of puma threads available at current scale           |
-| Gauge | `puma_max_threads`          | Number of puma threads at available at max scale            |
+| Type  | Name                        | Description                                                                                                         |
+| ---   | ---                         | ---                                                                                                                 |
+| Gauge | `puma_workers`              | Number of puma workers                                                                                              |
+| Gauge | `puma_booted_workers`       | Number of puma workers booted                                                                                       |
+| Gauge | `puma_old_workers`          | Number of old puma workers                                                                                          |
+| Gauge | `puma_running_threads`      | How many threads are spawned. A spawned thread may be busy processing a request or waiting for a new request        |
+| Gauge | `puma_request_backlog`      | Number of requests waiting to be processed by a puma thread                                                         |
+| Gauge | `puma_thread_pool_capacity` | Number of puma threads available at current scale                                                                   |
+| Gauge | `puma_max_threads`          | Number of puma threads at available at max scale                                                                    |
+| Gauge | `puma_busy_threads`         | Running - how many threads are waiting to receive work + how many requests are waiting for a thread to pick them up |
 
 All metrics may have a `phase` label and all custom labels provided with the `labels` option.
 

data/lib/prometheus_exporter/instrumentation/delayed_job.rb

@@ -40,6 +40,8 @@ module PrometheusExporter::Instrumentation
 
     def call(job, max_attempts, enqueued_count, pending_count, include_module_name, *args, &block)
       success = false
+      job_name = job.handler.to_s.match(JOB_CLASS_REGEXP).to_a[include_module_name ? 1 : 2].to_s
+      job_name ||= job.try(:name)
       start = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
       latency = Time.current - job.run_at
       attempts = job.attempts + 1 # Increment because we're adding the current attempt
@@ -51,7 +53,7 @@ module PrometheusExporter::Instrumentation
 
       @client.send_json(
         type: "delayed_job",
-        name: job.handler.to_s.match(JOB_CLASS_REGEXP).to_a[include_module_name ? 1 : 2].to_s,
+        name: job_name,
         queue_name: job.queue,
         success: success,
         duration: duration,

data/lib/prometheus_exporter/instrumentation/process.rb

@@ -62,6 +62,8 @@ module PrometheusExporter::Instrumentation
       metric[:rss] = rss
     end
 
+    SWEEPING_AND_MARKING = RUBY_VERSION >= "3.3.0"
+
     def collect_gc_stats(metric)
       stat = GC.stat
       metric[:heap_live_slots] = stat[:heap_live_slots]
@@ -71,6 +73,10 @@ module PrometheusExporter::Instrumentation
       metric[:allocated_objects_total] = stat[:total_allocated_objects]
       metric[:malloc_increase_bytes_limit] = stat[:malloc_increase_bytes_limit]
       metric[:oldmalloc_increase_bytes_limit] = stat[:oldmalloc_increase_bytes_limit]
+      if SWEEPING_AND_MARKING
+        metric[:marking_time] = stat[:marking_time]
+        metric[:sweeping_time] = stat[:sweeping_time]
+      end
     end
 
     def collect_v8_stats(metric)

data/lib/prometheus_exporter/instrumentation/puma.rb

@@ -61,11 +61,13 @@ module PrometheusExporter::Instrumentation
       metric[:running_threads] ||= 0
       metric[:thread_pool_capacity] ||= 0
       metric[:max_threads] ||= 0
+      metric[:busy_threads] ||= 0
 
       metric[:request_backlog] += status["backlog"]
       metric[:running_threads] += status["running"]
       metric[:thread_pool_capacity] += status["pool_capacity"]
       metric[:max_threads] += status["max_threads"]
+      metric[:busy_threads] += status["busy_threads"]
     end
   end
 end

data/lib/prometheus_exporter/middleware.rb

@@ -74,11 +74,11 @@ class PrometheusExporter::Middleware
   end
 
   def default_labels(env, result)
-    params = env["action_dispatch.request.parameters"]
+    controller_instance = env["action_controller.instance"]
     action = controller = nil
-    if params
-      action = params["action"]
-      controller = params["controller"]
+    if controller_instance
+      action = controller_instance.action_name
+      controller = controller_instance.controller_name
     elsif (cors = env["rack.cors"]) && cors.respond_to?(:preflight?) && cors.preflight?
       # if the Rack CORS Middleware identifies the request as a preflight request,
       # the stack doesn't get to the point where controllers/actions are defined

data/lib/prometheus_exporter/server/collector.rb

@@ -1,8 +1,13 @@
 # frozen_string_literal: true
 
+require "logger"
+
 module PrometheusExporter::Server
   class Collector < CollectorBase
-    def initialize(json_serializer: nil)
+    attr_reader :logger
+
+    def initialize(json_serializer: nil, logger: Logger.new(STDERR))
+      @logger = logger
       @process_metrics = []
       @metrics = {}
       @mutex = Mutex.new
@@ -40,6 +45,8 @@ module PrometheusExporter::Server
           metric = @metrics[obj["name"]]
           metric = register_metric_unsafe(obj) if !metric
 
+          next unless metric
+
           keys = obj["keys"] || {}
           keys = obj["custom_labels"].merge(keys) if obj["custom_labels"]
 
@@ -74,6 +81,11 @@ module PrometheusExporter::Server
       help = obj["help"]
       opts = symbolize_keys(obj["opts"] || {})
 
+      if !name
+        logger.warn "failed to register metric due to empty name #{obj}"
+        return
+      end
+
       metric =
         case obj["type"]
         when "gauge"
@@ -89,7 +101,7 @@ module PrometheusExporter::Server
       if metric
         @metrics[name] = metric
       else
-        STDERR.puts "failed to register metric #{obj}"
+        logger.warn "failed to register metric #{obj}"
       end
     end
 

data/lib/prometheus_exporter/server/process_collector.rb

@@ -16,6 +16,8 @@ module PrometheusExporter::Server
         "Limit before Ruby triggers a GC against current objects (bytes).",
       oldmalloc_increase_bytes_limit:
         "Limit before Ruby triggers a major GC against old objects (bytes).",
+      marking_time: "Time spent in GC marking.",
+      sweeping_time: "Time spent in GC sweeping.",
     }
 
     PROCESS_COUNTERS = {

data/lib/prometheus_exporter/server/puma_collector.rb

@@ -13,6 +13,13 @@ module PrometheusExporter::Server
       max_threads: "Number of puma threads at available at max scale.",
     }
 
+    if defined?(::Puma::Const) &&
+         Gem::Version.new(::Puma::Const::VERSION) >= Gem::Version.new("6.6.0")
+      PUMA_GAUGES[
+        :busy_threads
+      ] = "Wholistic stat reflecting the overall current state of work to be done and the capacity to do it"
+    end
+
     def initialize
       @puma_metrics = MetricsContainer.new(ttl: MAX_PUMA_METRIC_AGE)
       @puma_metrics.filter = ->(new_metric, old_metric) do

data/lib/prometheus_exporter/server/runner.rb

@@ -50,7 +50,7 @@ module PrometheusExporter::Server
         )
       end
 
-      server =
+      @server =
         server_class.new(
           port: port,
           bind: bind,
@@ -60,7 +60,11 @@ module PrometheusExporter::Server
           auth: auth,
           realm: realm,
         )
-      server.start
+      @server.start
+    end
+
+    def stop
+      @server.stop
     end
 
     attr_accessor :unicorn_listen_address, :unicorn_pid_file