yabeda-sidekiq 0.9.0 → 0.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f287f1d10d316b33e5d56ea36a9d8dbcc9b1f520b0d3d21d72f11dd5bc40e90f
-  data.tar.gz: a3809269e4d6011f7dab4ce89391fd2d14fc0775c1dac768a8105f1949a0205b
+  metadata.gz: ad8acaec8b59c1c088c6c924d5bbc777b1c87c346b56389045e4609b3920af11
+  data.tar.gz: 26e87dbce002e1053eb6f5c709a78642b002691cad1f69e7d1d02722dd692790
 SHA512:
-  metadata.gz: 3cb86100524d1b8457caad978f1fcaaf299f27e27c74f787f8b9059b02682f6c4fd04017ad60f9e2b22c46d7d64d18b18cac935fea2586a38f886bd47a31e40c
-  data.tar.gz: e536ccdca8b3a9e66eb6ebf3e3795e8083aa46295f9587f5d80e23bf3ab7634851719410d28518d843d44a9c68904e7b162258da1f94737892a6f38c5f69b5e3
+  metadata.gz: 880a85b150e20c3b78a330d7fa9a2ae578d7fba140adaf9d01e8fdae467c57f746607efa601404b9b9f20d8bbf0fb98b560177ef198095fa48176f79ccc5ce10
+  data.tar.gz: 720157c2e892d40ddd776e186055403d79162a8554c36a080003678f1f73d69a9cf924c5ef17bab42235b309506ef620eca80c6311e9af90f83b5c9460585d5a

data/.github/workflows/lint.yml

@@ -15,10 +15,10 @@ jobs:
     name: RuboCop
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - uses: ruby/setup-ruby@v1
         with:
-          ruby-version: "3.0"
+          ruby-version: "3.2"
           bundler-cache: true
       - name: Lint Ruby code with RuboCop
         run: |

data/.github/workflows/{build-release.yml → release.yml}

@@ -1,4 +1,4 @@
-name: Build and release gem to RubyGems
+name: Build and release gem
 
 on:
   push:
@@ -8,13 +8,17 @@ on:
 jobs:
   release:
     runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      id-token: write
+      packages: write
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0 # Fetch current tag as annotated. See https://github.com/actions/checkout/issues/290
       - uses: ruby/setup-ruby@v1
         with:
-          ruby-version: 2.7
+          ruby-version: "3.3"
       - name: "Extract data from tag: version, message, body"
         id: tag
         run: |
@@ -75,8 +79,8 @@ jobs:
           GEM_HOST_API_KEY: Bearer ${{ secrets.GITHUB_TOKEN }}
         run: |
           gem push yabeda-sidekiq-${{ steps.tag.outputs.version }}.gem --host https://rubygems.pkg.github.com/${{ github.repository_owner }}
+      - name: Configure RubyGems Credentials
+        uses: rubygems/configure-rubygems-credentials@main
       - name: Publish to RubyGems
-        env:
-          GEM_HOST_API_KEY: "${{ secrets.RUBYGEMS_API_KEY }}"
         run: |
           gem push yabeda-sidekiq-${{ steps.tag.outputs.version }}.gem

data/.github/workflows/test.yml

@@ -10,7 +10,7 @@ on:
 
 jobs:
   test:
-    name: 'Ruby ${{ matrix.ruby }}'
+    name: 'Ruby ${{ matrix.ruby }} × Sidekiq v${{ matrix.sidekiq }} × ActiveJob v${{ matrix.activejob }}'
     # Skip running tests for local pull requests (use push event instead), run only for foreign ones
     if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login != github.event.pull_request.base.repo.owner.login
     runs-on: ubuntu-latest
@@ -18,30 +18,26 @@ jobs:
       fail-fast: false
       matrix:
         include:
+          - ruby: '3.3'
+            sidekiq: '7'
+            activejob: '7.1'
+          - ruby: '3.2'
+            sidekiq: '7'
+            activejob: '7.0'
           - ruby: '3.1'
+            sidekiq: '6'
+            activejob: '6.1'
           - ruby: '3.0'
-          - ruby: '2.7'
-          - ruby: '2.6'
-          - ruby: '2.5'
-    container:
-      image: ruby:${{ matrix.ruby }}
-      env:
-        CI: true
+            sidekiq: '5'
+            activejob: '6.0'
+    env:
+      SIDEKIQ_VERSION: '${{ matrix.sidekiq }}'
+      ACTIVEJOB_VERSION: '${{ matrix.activejob }}'
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v2
+      - uses: actions/checkout@v4
+      - uses: ruby/setup-ruby@v1
         with:
-          path: vendor/bundle
-          key: bundle-${{ matrix.ruby }}-${{ hashFiles('**/*.gemspec') }}-${{ hashFiles('**/Gemfile') }}
-          restore-keys: |
-            bundle-${{ matrix.ruby }}-${{ hashFiles('**/*.gemspec') }}-${{ hashFiles('**/Gemfile') }}
-            bundle-${{ matrix.ruby }}-
-      - name: Upgrade Bundler to 2.0 (for older Rubies)
-        run: gem install bundler -v '~> 2.0'
-      - name: Bundle install
-        run: |
-          bundle config path vendor/bundle
-          bundle install
-          bundle update
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
       - name: Run RSpec
         run: bundle exec rspec

data/.rubocop.yml

@@ -3,7 +3,7 @@ require:
   - rubocop-rspec
 
 AllCops:
-  TargetRubyVersion: 2.3
+  TargetRubyVersion: 2.5
 
 Metrics/BlockLength:
   Enabled: false
@@ -65,3 +65,15 @@ Metrics/MethodLength:
 
 Metrics/AbcSize:
   Max: 17
+
+Style/SoleNestedConditional:
+  Enabled: false
+
+Style/ExplicitBlockArgument:
+  Enabled: false
+
+Gemspec/RequiredRubyVersion:
+  Enabled: false
+
+Metrics/ModuleLength:
+  Max: 200

data/CHANGELOG.md

@@ -7,6 +7,26 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## Unreleased
 
+## 0.11.0 - 2024-02-07
+
+### Added
+
+- `retries_segmented_by_queue` configuration setting to allow segmentation of retry count by queue.
+
+  It is disabled by default as it requires to iterate over all jobs in the retry set and may be very slow if number of retries is huge.
+
+  Pull request [#32](https://github.com/yabeda-rb/yabeda-sidekiq/pull/32) by [@SxDx]
+
+## 0.10.0 - 2022-10-25
+
+### Added
+
+- New metric `sidekiq_jobs_rerouted_total_count` to measure jobs that on enqueue were pushed to different queue from the one specified in worker's `sidekiq_options`. See [#30](https://github.com/yabeda-rb/yabeda-sidekiq/pull/30). [@LukinEgor]
+
+### Fixed
+
+- In `sidekiq_jobs_enqueued_total_count` track real queue that job was pushed into, not the one specified in `sidekiq_options` (sometimes they may be different). See [#30](https://github.com/yabeda-rb/yabeda-sidekiq/pull/30). [@LukinEgor]
+
 ## 0.9.0 - 2022-09-26
 
 ### Added
@@ -108,3 +128,5 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 [@dsalahutdinov]: https://github.com/dsalahutdinov "Salahutdinov Dmitry"
 [@asusikov]: https://github.com/asusikov "Alexander Susikov"
 [@mrexox]: https://github.com/mrexox "Valentine Kiselev"
+[@LukinEgor]: https://github.com/LukinEgor "Egor Lukin"
+[@SxDx]: https://github.com/SxDx "René Koller"

data/Gemfile

@@ -7,10 +7,36 @@ git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
 # Specify your gem's dependencies in yabeda-sidekiq.gemspec
 gemspec
 
+# rubocop:disable Bundler/DuplicatedGem
+sidekiq_version = ENV.fetch("SIDEKIQ_VERSION", "~> 7.2")
+case sidekiq_version
+when "HEAD"
+  gem "sidekiq", git: "https://github.com/sidekiq/sidekiq.git"
+else
+  sidekiq_version = "~> #{sidekiq_version}.0" if sidekiq_version.match?(/^\d+(?:\.\d+)?$/)
+  gem "sidekiq", sidekiq_version
+end
+
+activejob_version = ENV.fetch("ACTIVEJOB_VERSION", "~> 7.1")
+case activejob_version
+when "HEAD"
+  git "https://github.com/rails/rails.git" do
+    gem "activejob"
+    gem "activesupport"
+    gem "rails"
+  end
+else
+  activejob_version = "~> #{activejob_version}.0" if activejob_version.match?(/^\d+\.\d+$/)
+  gem "activejob", activejob_version
+  gem "activesupport", activejob_version
+end
+# rubocop:enable Bundler/DuplicatedGem
+
 group :development, :test do
   gem "pry"
   gem "pry-byebug", platform: :mri
 
-  gem "rubocop", "~> 0.80.0"
+  gem "yabeda", github: "yabeda-rb/yabeda", branch: "master" # For RSpec matchers
+  gem "rubocop", "~> 1.0"
   gem "rubocop-rspec"
 end

data/README.md

@@ -62,6 +62,16 @@ Metrics representing state of the whole Sidekiq installation (queues, processes,
 
 By default all sidekiq worker processes (servers) collects global metrics about whole Sidekiq installation. This can be overridden by setting `collect_cluster_metrics` config key to `true` for non-Sidekiq processes or to `false` for Sidekiq processes (e.g. by setting `YABEDA_SIDEKIQ_COLLECT_CLUSTER_METRICS` env variable to `no`, see other methods in [anyway_config] docs).
 
+### Client metrics
+
+Metrics collected where jobs are being pushed to queues (everywhere):
+
+- Total number of enqueued jobs: `sidekiq_jobs_enqueued_total_count` (segmented by `queue` and `worker` class name)
+
+- Total number of rerouted jobs: `sidekiq_jobs_rerouted_total_count` (segmented by origin queue `from_queue`, rerouted queue `to_queue`, and `worker` class name).
+
+  Rerouted jobs are jobs that on enqueue were pushed to different queue from the one specified in worker's `sidekiq_options`, most probably by some middleware.
+
 ## Custom tags
 
 You can add additional tags to these metrics by declaring `yabeda_tags` method in your worker.
@@ -89,13 +99,15 @@ end
 
 Configuration is handled by [anyway_config] gem. With it you can load settings from environment variables (upcased and prefixed with `YABEDA_SIDEKIQ_`), YAML files, and other sources. See [anyway_config] docs for details.
 
-Config key                | Type     | Default                                                 | Description |
-------------------------- | -------- | ------------------------------------------------------- | ----------- |
-`collect_cluster_metrics` | boolean  | Enabled in Sidekiq worker processes, disabled otherwise | Defines whether this Ruby process should collect and expose metrics representing state of the whole Sidekiq installation (queues, processes, etc). |
+Config key                   | Type     | Default                                                 | Description                                                                                                                                        |
+---------------------------- | -------- | ------------------------------------------------------- |----------------------------------------------------------------------------------------------------------------------------------------------------|
+`collect_cluster_metrics`    | boolean  | Enabled in Sidekiq worker processes, disabled otherwise | Defines whether this Ruby process should collect and expose metrics representing state of the whole Sidekiq installation (queues, processes, etc). |
+`declare_process_metrics`    | boolean  | Enabled in Sidekiq worker processes, disabled otherwise | Declare metrics that are only tracked inside worker process even outside of them. Useful for multiprocess metric collection.                       |
+`retries_segmented_by_queue` | boolean  | Disabled                                                | Defines wheter retries are segemented by queue or reported as a single metric                                                                      |
 
 # Roadmap (TODO or Help wanted)
 
- - Implement optional segmentation of retry/schedule/dead sets
+ - Implement optional segmentation of schedule/dead sets
 
    It should be disabled by default as it requires to iterate over all jobs in sets and may be very slow on large sets.
 

data/lib/yabeda/sidekiq/client_middleware.rb

@@ -5,8 +5,14 @@ module Yabeda
     # Client middleware to count number of enqueued jobs
     class ClientMiddleware
       def call(worker, job, queue, _redis_pool)
-        labels = Yabeda::Sidekiq.labelize(worker, job, queue)
+        labels = Yabeda::Sidekiq.labelize(worker, job, job["queue"] || queue)
         Yabeda.sidekiq_jobs_enqueued_total.increment(labels)
+
+        if job["queue"] && job["queue"] != queue
+          labels = Yabeda::Sidekiq.labelize(worker, job, queue)
+          Yabeda.sidekiq_jobs_rerouted_total.increment({ from_queue: queue, to_queue: job["queue"], **labels.except(:queue) })
+        end
+
         yield
       end
     end

data/lib/yabeda/sidekiq/config.rb

@@ -16,6 +16,10 @@ module Yabeda
 
       # Declare metrics that are only tracked inside worker process even outside them
       attr_config declare_process_metrics: ::Sidekiq.server?
+
+      # Retries are tracked by default as a single metric. If you want to track them separately for each queue, set this to +true+
+      # Disabled by default because it is quite slow if the retry set is large
+      attr_config retries_segmented_by_queue: false
     end
   end
 end

data/lib/yabeda/sidekiq/version.rb

@@ -2,6 +2,6 @@
 
 module Yabeda
   module Sidekiq
-    VERSION = "0.9.0"
+    VERSION = "0.11.0"
   end
 end

data/lib/yabeda/sidekiq.rb

@@ -26,6 +26,7 @@ module Yabeda
       group :sidekiq
 
       counter :jobs_enqueued_total, tags: %i[queue worker], comment: "A counter of the total number of jobs sidekiq enqueued."
+      counter :jobs_rerouted_total, tags: %i[from_queue to_queue worker], comment: "A counter of the total number of rerouted jobs sidekiq enqueued."
 
       if config.declare_process_metrics # defaults to +::Sidekiq.server?+
         counter   :jobs_executed_total,  tags: %i[queue worker], comment: "A counter of the total number of jobs sidekiq executed."
@@ -48,13 +49,17 @@ module Yabeda
       # Metrics not specific for current Sidekiq process, but representing state of the whole Sidekiq installation (queues, processes, etc)
       # You can opt-out from collecting these by setting YABEDA_SIDEKIQ_COLLECT_CLUSTER_METRICS to falsy value (+no+ or +false+)
       if config.collect_cluster_metrics # defaults to +::Sidekiq.server?+
-        gauge     :jobs_waiting_count,   tags: %i[queue], aggregation: :most_recent, comment: "The number of jobs waiting to process in sidekiq."
-        gauge     :active_workers_count, tags: [],        aggregation: :most_recent, comment: "The number of currently running machines with sidekiq workers."
-        gauge     :jobs_scheduled_count, tags: [],        aggregation: :most_recent, comment: "The number of jobs scheduled for later execution."
-        gauge     :jobs_retry_count,     tags: [],        aggregation: :most_recent, comment: "The number of failed jobs waiting to be retried"
-        gauge     :jobs_dead_count,      tags: [],        aggregation: :most_recent, comment: "The number of jobs exceeded their retry count."
-        gauge     :active_processes,     tags: [],        aggregation: :most_recent, comment: "The number of active Sidekiq worker processes."
-        gauge     :queue_latency,        tags: %i[queue], aggregation: :most_recent, comment: "The queue latency, the difference in seconds since the oldest job in the queue was enqueued"
+        retry_count_tags = config.retries_segmented_by_queue ? %i[queue] : []
+
+        gauge     :jobs_waiting_count,   tags: %i[queue],        aggregation: :most_recent, comment: "The number of jobs waiting to process in sidekiq."
+        gauge     :active_workers_count, tags: [],               aggregation: :most_recent,
+                                         comment: "The number of currently running machines with sidekiq workers."
+        gauge     :jobs_scheduled_count, tags: [],               aggregation: :most_recent, comment: "The number of jobs scheduled for later execution."
+        gauge     :jobs_retry_count,     tags: retry_count_tags, aggregation: :most_recent, comment: "The number of failed jobs waiting to be retried"
+        gauge     :jobs_dead_count,      tags: [],               aggregation: :most_recent, comment: "The number of jobs exceeded their retry count."
+        gauge     :active_processes,     tags: [],               aggregation: :most_recent, comment: "The number of active Sidekiq worker processes."
+        gauge     :queue_latency,        tags: %i[queue],        aggregation: :most_recent,
+                                         comment: "The queue latency, the difference in seconds since the oldest job in the queue was enqueued"
       end
 
       collect do
@@ -71,21 +76,22 @@ module Yabeda
         sidekiq_jobs_scheduled_count.set({}, stats.scheduled_size)
         sidekiq_jobs_dead_count.set({}, stats.dead_size)
         sidekiq_active_processes.set({}, stats.processes_size)
-        sidekiq_jobs_retry_count.set({}, stats.retry_size)
 
         ::Sidekiq::Queue.all.each do |queue|
           sidekiq_queue_latency.set({ queue: queue.name }, queue.latency)
         end
 
-        # That is quite slow if your retry set is large
-        # I don't want to enable it by default
-        # retries_by_queues =
-        #     ::Sidekiq::RetrySet.new.each_with_object(Hash.new(0)) do |job, cntr|
-        #       cntr[job["queue"]] += 1
-        #     end
-        # retries_by_queues.each do |queue, count|
-        #   sidekiq_jobs_retry_count.set({ queue: queue }, count)
-        # end
+        if config.retries_segmented_by_queue
+          retries_by_queues =
+            ::Sidekiq::RetrySet.new.each_with_object(Hash.new(0)) do |job, cntr|
+              cntr[job["queue"]] += 1
+            end
+          retries_by_queues.each do |queue, count|
+            sidekiq_jobs_retry_count.set({ queue: queue }, count)
+          end
+        else
+          sidekiq_jobs_retry_count.set({}, stats.retry_size)
+        end
       end
     end
 
@@ -110,11 +116,7 @@ module Yabeda
       end
 
       def worker_class(worker, job)
-        if defined?(ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper)
-          if worker.is_a?(ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper) || worker == ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper
-            return job["wrapped"].to_s
-          end
-        end
+        worker = job["wrapped"] || worker
         (worker.is_a?(String) || worker.is_a?(Class) ? worker : worker.class).to_s
       end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: yabeda-sidekiq
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 0.11.0
 platform: ruby
 authors:
 - Andrey Novikov
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-09-26 00:00:00.000000000 Z
+date: 2024-02-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: anyway_config
@@ -122,8 +122,8 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".github/workflows/build-release.yml"
 - ".github/workflows/lint.yml"
+- ".github/workflows/release.yml"
 - ".github/workflows/test.yml"
 - ".gitignore"
 - ".rspec"
@@ -161,7 +161,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.5.3
 signing_key: 
 specification_version: 4
 summary: Extensible Prometheus exporter for monitoring your Sidekiq