yabeda-sidekiq 0.8.2 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '08af80462b44bb1631a4e7103354bf2cd3613807b629d282d31913ec8c2dfc75'
-  data.tar.gz: 60634a95cc22cbeaff73e0986651956e5348315cb42077f4c6a86447dfd45acc
+  metadata.gz: 6d63d31846975b74ec0fc5a7209b8b405065c52ff49074bcad1f36d92bb97e36
+  data.tar.gz: 3bd52a1b15a38d2fb80df6e54d9b702311e9311f37b04f5622e6c2b92411b768
 SHA512:
-  metadata.gz: 6f880ce4e139f01b07d375e62e252fa990d198e09838aa9d2a3b3d348984adda38debe8ddb33ccc8dd514a2f502f89c27f5acdfafe320479d59a397ac7e56000
-  data.tar.gz: 29c28c92077f5b3edbd1e844c6ddcba0bb4de99ca164dcfadca30a1c2369979a2cc3ac51192ee700b8d71fc82d68372fc4b12cf00521ae0ef822b5c08d2b9cb0
+  metadata.gz: '05919789943b693dcc75c7b3b50a6594b924f4703affdffac32720fae69b8cb1de0e3a9818301c16edc8d7b08c57e6431474ebce66d81fa5ca5a33805846ea46'
+  data.tar.gz: 17699edb8dc44c6ee0a7a89c0488253af8dbbb978f012e8d5c828c180e59644c82092f03dad5c517cf973c79853d167ec56c1da266b0e2de6b1c555b7ce05876

data/CHANGELOG.md

@@ -7,6 +7,28 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## Unreleased
 
+## 0.10.0 - 2022-10-25
+
+### Added
+
+- New metric `sidekiq_jobs_rerouted_total_count` to measure jobs that on enqueue were pushed to different queue from the one specified in worker's `sidekiq_options`. See [#30](https://github.com/yabeda-rb/yabeda-sidekiq/pull/30). [@LukinEgor]
+
+### Fixed
+
+- In `sidekiq_jobs_enqueued_total_count` track real queue that job was pushed into, not the one specified in `sidekiq_options` (sometimes they may be different). See [#30](https://github.com/yabeda-rb/yabeda-sidekiq/pull/30). [@LukinEgor]
+
+## 0.9.0 - 2022-09-26
+
+### Added
+
+- Configuration setting to declare worker in-process metrics outside workers.
+
+  It can be needed for official Prometheus client in multi-process mode where separate process expose metrics from Sidekiq worker processes.
+
+- `most_recent` aggregation for all cluster-wide gauges.
+
+  It is also needed for official Prometheus client in multi-process mode to reduce number of time series.
+
 ## 0.8.2 - 2022-09-14
 
 ### Added
@@ -96,3 +118,4 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 [@dsalahutdinov]: https://github.com/dsalahutdinov "Salahutdinov Dmitry"
 [@asusikov]: https://github.com/asusikov "Alexander Susikov"
 [@mrexox]: https://github.com/mrexox "Valentine Kiselev"
+[@LukinEgor]: https://github.com/LukinEgor "Egor Lukin"

data/README.md

@@ -62,6 +62,16 @@ Metrics representing state of the whole Sidekiq installation (queues, processes,
 
 By default all sidekiq worker processes (servers) collects global metrics about whole Sidekiq installation. This can be overridden by setting `collect_cluster_metrics` config key to `true` for non-Sidekiq processes or to `false` for Sidekiq processes (e.g. by setting `YABEDA_SIDEKIQ_COLLECT_CLUSTER_METRICS` env variable to `no`, see other methods in [anyway_config] docs).
 
+### Client metrics
+
+Metrics collected where jobs are being pushed to queues (everywhere):
+
+- Total number of enqueued jobs: `sidekiq_jobs_enqueued_total_count` (segmented by `queue` and `worker` class name)
+
+- Total number of rerouted jobs: `sidekiq_jobs_rerouted_total_count` (segmented by origin queue `from_queue`, rerouted queue `to_queue`, and `worker` class name).
+
+  Rerouted jobs are jobs that on enqueue were pushed to different queue from the one specified in worker's `sidekiq_options`, most probably by some middleware.
+
 ## Custom tags
 
 You can add additional tags to these metrics by declaring `yabeda_tags` method in your worker.
@@ -89,9 +99,10 @@ end
 
 Configuration is handled by [anyway_config] gem. With it you can load settings from environment variables (upcased and prefixed with `YABEDA_SIDEKIQ_`), YAML files, and other sources. See [anyway_config] docs for details.
 
-Config key                | Type     | Default                                                 | Description |
-------------------------- | -------- | ------------------------------------------------------- | ----------- |
+Config key                | Type     | Default                                                 | Description                                                                                                                                        |
+------------------------- | -------- | ------------------------------------------------------- |----------------------------------------------------------------------------------------------------------------------------------------------------|
 `collect_cluster_metrics` | boolean  | Enabled in Sidekiq worker processes, disabled otherwise | Defines whether this Ruby process should collect and expose metrics representing state of the whole Sidekiq installation (queues, processes, etc). |
+`declare_process_metrics` | boolean  | Enabled in Sidekiq worker processes, disabled otherwise | Declare metrics that are only tracked inside worker process even outside of them. Useful for multiprocess metric collection.                       |
 
 # Roadmap (TODO or Help wanted)
 

data/lib/yabeda/sidekiq/client_middleware.rb

@@ -5,8 +5,14 @@ module Yabeda
     # Client middleware to count number of enqueued jobs
     class ClientMiddleware
       def call(worker, job, queue, _redis_pool)
-        labels = Yabeda::Sidekiq.labelize(worker, job, queue)
+        labels = Yabeda::Sidekiq.labelize(worker, job, job["queue"] || queue)
         Yabeda.sidekiq_jobs_enqueued_total.increment(labels)
+
+        if job["queue"] && job["queue"] != queue
+          labels = Yabeda::Sidekiq.labelize(worker, job, queue)
+          Yabeda.sidekiq_jobs_rerouted_total.increment({ from_queue: queue, to_queue: job["queue"], **labels.except(:queue) })
+        end
+
         yield
       end
     end

data/lib/yabeda/sidekiq/config.rb

@@ -13,6 +13,9 @@ module Yabeda
       #  - force disable if you don't want multiple Sidekiq workers to report the same numbers (that causes excess load to both Redis and monitoring)
       #  - force enable if you want non-Sidekiq process to collect them (like dedicated metric exporter process)
       attr_config collect_cluster_metrics: ::Sidekiq.server?
+
+      # Declare metrics that are only tracked inside worker process even outside them
+      attr_config declare_process_metrics: ::Sidekiq.server?
     end
   end
 end

data/lib/yabeda/sidekiq/version.rb

@@ -2,6 +2,6 @@
 
 module Yabeda
   module Sidekiq
-    VERSION = "0.8.2"
+    VERSION = "0.10.0"
   end
 end

data/lib/yabeda/sidekiq.rb

@@ -26,8 +26,9 @@ module Yabeda
       group :sidekiq
 
       counter :jobs_enqueued_total, tags: %i[queue worker], comment: "A counter of the total number of jobs sidekiq enqueued."
+      counter :jobs_rerouted_total, tags: %i[from_queue to_queue worker], comment: "A counter of the total number of rerouted jobs sidekiq enqueued."
 
-      if ::Sidekiq.server?
+      if config.declare_process_metrics # defaults to +::Sidekiq.server?+
         counter   :jobs_executed_total,  tags: %i[queue worker], comment: "A counter of the total number of jobs sidekiq executed."
         counter   :jobs_success_total,   tags: %i[queue worker], comment: "A counter of the total number of jobs successfully processed by sidekiq."
         counter   :jobs_failed_total,    tags: %i[queue worker], comment: "A counter of the total number of jobs failed in sidekiq."
@@ -48,13 +49,14 @@ module Yabeda
       # Metrics not specific for current Sidekiq process, but representing state of the whole Sidekiq installation (queues, processes, etc)
       # You can opt-out from collecting these by setting YABEDA_SIDEKIQ_COLLECT_CLUSTER_METRICS to falsy value (+no+ or +false+)
       if config.collect_cluster_metrics # defaults to +::Sidekiq.server?+
-        gauge     :jobs_waiting_count,   tags: %i[queue], comment: "The number of jobs waiting to process in sidekiq."
-        gauge     :active_workers_count, tags: [],        comment: "The number of currently running machines with sidekiq workers."
-        gauge     :jobs_scheduled_count, tags: [],        comment: "The number of jobs scheduled for later execution."
-        gauge     :jobs_retry_count,     tags: [],        comment: "The number of failed jobs waiting to be retried"
-        gauge     :jobs_dead_count,      tags: [],        comment: "The number of jobs exceeded their retry count."
-        gauge     :active_processes,     tags: [],        comment: "The number of active Sidekiq worker processes."
-        gauge     :queue_latency,        tags: %i[queue], comment: "The queue latency, the difference in seconds since the oldest job in the queue was enqueued"
+        gauge     :jobs_waiting_count,   tags: %i[queue], aggregation: :most_recent, comment: "The number of jobs waiting to process in sidekiq."
+        gauge     :active_workers_count, tags: [],        aggregation: :most_recent, comment: "The number of currently running machines with sidekiq workers."
+        gauge     :jobs_scheduled_count, tags: [],        aggregation: :most_recent, comment: "The number of jobs scheduled for later execution."
+        gauge     :jobs_retry_count,     tags: [],        aggregation: :most_recent, comment: "The number of failed jobs waiting to be retried"
+        gauge     :jobs_dead_count,      tags: [],        aggregation: :most_recent, comment: "The number of jobs exceeded their retry count."
+        gauge     :active_processes,     tags: [],        aggregation: :most_recent, comment: "The number of active Sidekiq worker processes."
+        gauge     :queue_latency,        tags: %i[queue], aggregation: :most_recent,
+                                         comment: "The queue latency, the difference in seconds since the oldest job in the queue was enqueued"
       end
 
       collect do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: yabeda-sidekiq
 version: !ruby/object:Gem::Version
-  version: 0.8.2
+  version: 0.10.0
 platform: ruby
 authors:
 - Andrey Novikov
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-09-14 00:00:00.000000000 Z
+date: 2022-10-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: anyway_config