sidekiq-instrument 0.6.1 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2a5bba582f4b34766f77e4c9fb0a7c60e026111186fd3807f23c027f421e420d
-  data.tar.gz: 6404eabed92c2b7e6a2fc4c125dcc9e0b81ab35e2db1a98a9175fcb533a0bd3b
+  metadata.gz: c7de17bf15cf561935e482b332b89974f84409e0c1bfa00a544503ce95bcfd8d
+  data.tar.gz: 1732bbfc31fc7c04fe066613845f9e81917873f2fc25ae93ec960aba611c347d
 SHA512:
-  metadata.gz: 458c593c2cce45ec37a5c659b32d340f1244c0eed14d3c3bb9286d9b4208d095c4ab988b2c4761dbd4499b1f5a68ddf813b223536bc2dcbfcc055ceacd418728
-  data.tar.gz: 1c825fa318f119418d6b93bf3511f433451bb004281bfc1131c553e57e0f421adc0d0b43f211aa59ea21b5d0846cc123288068c49eb96fecc26ff3433010eba0
+  metadata.gz: 65c1f85a4c9a13b4cd2c3585712af41b07a8979f2a736c44d971fc5175d737f7a7cb6104d6383af714d94273a41e017bae361e29f7381587294d463432bf7551
+  data.tar.gz: b1f25cab5729fe6bcb76a81556fc18583ab800b3dd2a66c50c9fea3834abcf1a422aabb06bc6d5ef39b69be4675ab35e1dab9c53e43920e377888fd91aa79b2d

data/README.md

@@ -84,6 +84,12 @@ For each job, the following metrics will be reported:
 4. **shared.sidekiq._queue_._job_.error**: counter incremented each time a
    job fails.
 
+For job retry attempts, the above 4 metrics will still be reported but the enqueue/dequeue metrics
+will have a `.retry` appended:
+
+1. **shared.sidekiq._queue_._job_.enqueue.retry**
+2. **shared.sidekiq._queue_._job_.dequeue.retry**
+
 The metric names can be changed by overriding the `statsd_metric_name`
 method in your worker classes.
 
@@ -92,7 +98,7 @@ For each queue, the following metrics will be reported:
 2. **shared.sidekiq._queue_.latency**: gauge of how long the oldest job has been in the queue
 
 For each worker, the following metrics and tags will be reported:
-1. **sidekiq.worker_metrics.inqueue.#{key}**: number of jobs "in queue" per worker, uses redis to track increment/decrement
+1. **sidekiq.worker_metrics.in_queue.#{key}**: number of jobs "in queue" per worker, uses redis to track increment/decrement (**this metric is currently inaccurate**)
 
 ## DogStatsD Keys
 For each job, the following metrics and tags will be reported:
@@ -106,14 +112,23 @@ For each job, the following metrics and tags will be reported:
 4. **sidekiq.error (tags: {queue: _queue_, worker: _job_})**: counter incremented each time a
    job fails.
 
+For job retry attempts, the above 4 metrics will still be reported but the enqueue/dequeue metrics
+will have a `.retry` appended:
+
+1. **sidekiq.enqueue.retry (tags: {queue: _queue_, worker: _job_})**
+2. **sidekiq.dequeue.retry (tags: {queue: _queue_, worker: _job_})**
+
 For each queue, the following metrics and tags will be reported:
 1. **sidekiq.queue.size (tags: {queue: _queue_})**: gauge of how many jobs are in the queue
 2. **sidekiq.queue.latency (tags: {queue: _queue_})**: gauge of how long the oldest job has been in the queue
 
 For each worker, the following metrics and tags will be reported:
-1. **sidekiq.worker_metrics.inqueue.#{key}**: number of jobs "in queue" per worker, uses redis to track increment/decrement
+1. **sidekiq.worker_metrics.in_queue.#{key}**: number of jobs "in queue" per worker, uses redis to track increment/decrement (**this metric is currently inaccurate**)
 
 ## Worker
+
+**WARNING: The metrics reported by this Worker are currently inaccurate.**
+
 There is a worker, `Sidekiq::Instrument::Worker`, that submits gauges
 for various interesting statistics; namely, the bulk of the information in `Sidekiq::Stats`
 and the sizes of each individual queue. While the worker class is a fully valid Sidekiq worker,

data/lib/sidekiq/instrument/middleware/client.rb

@@ -7,16 +7,24 @@ module Sidekiq::Instrument
   class ClientMiddleware
     include Sidekiq::Instrument::MetricNames
 
-    def call(worker_class, job, queue, redis_pool)
+    def call(worker_class, job, queue, _redis_pool)
       # worker_class is a const in sidekiq >= 6.x
       klass = Object.const_get(worker_class.to_s)
       class_instance = klass.new
-      Statter.statsd.increment(metric_name(class_instance, 'enqueue'))
-      Statter.dogstatsd&.increment('sidekiq.enqueue', worker_dog_options(class_instance))
-      WorkerMetrics.trace_workers_increment_counter(klass.name.underscore, redis_pool)
-      result = yield
+
+      # This is needed because the ClientMiddleware is called twice for scheduled jobs
+      # - Once when it gets scheduled
+      # - Once when it gets dequeued for processing
+      # We only want to increment the enqueue metric when the job is scheduled and
+      # Sidekiq::Context.current[:class] is only ever set when the job is scheduled
+      if Sidekiq::Context.current[:class].present?
+        WorkerMetrics.trace_workers_increment_counter(klass.name.underscore)
+        Statter.statsd.increment(metric_name(class_instance, 'enqueue'))
+        Statter.dogstatsd&.increment('sidekiq.enqueue', worker_dog_options(class_instance))
+      end
+
       Statter.dogstatsd&.flush(sync: true)
-      result
+      yield
     end
   end
 end

data/lib/sidekiq/instrument/middleware/server.rb

@@ -7,22 +7,40 @@ module Sidekiq::Instrument
   class ServerMiddleware
     include Sidekiq::Instrument::MetricNames
 
-    def call(worker, job, queue, &block)
-      Statter.statsd.increment(metric_name(worker, 'dequeue'))
-      Statter.dogstatsd&.increment('sidekiq.dequeue', worker_dog_options(worker))
+    def call(worker, job, _queue, &block)
+      dequeue_string = is_retry(job) ? 'dequeue.retry' : 'dequeue'
+      Statter.dogstatsd&.increment("sidekiq.#{dequeue_string}", worker_dog_options(worker))
+      Statter.statsd.increment(metric_name(worker, dequeue_string))
 
       start_time = Time.now
-      WorkerMetrics.trace_workers_decrement_counter(worker.class.to_s.underscore)
       yield block
       execution_time_ms = (Time.now - start_time) * 1000
       Statter.statsd.measure(metric_name(worker, 'runtime'), execution_time_ms)
       Statter.dogstatsd&.timing('sidekiq.runtime', execution_time_ms, worker_dog_options(worker))
     rescue StandardError => e
+      # if we have retries left, increment the enqueue.retry counter to indicate the job is going back on the queue
+      if max_retries(worker) > current_retries(job) + 1
+        WorkerMetrics.trace_workers_increment_counter(worker.class.to_s.underscore)
+        Statter.dogstatsd&.increment('sidekiq.enqueue.retry', worker_dog_options(worker))
+      end
+
       Statter.statsd.increment(metric_name(worker, 'error'))
       Statter.dogstatsd&.increment('sidekiq.error', worker_dog_options(worker))
       raise e
     ensure
+      WorkerMetrics.trace_workers_decrement_counter(worker.class.to_s.underscore)
       Statter.dogstatsd&.flush(sync: true)
     end
+
+    private
+
+    # returns -1 if no retries have been attempted
+    def current_retries(job)
+      job["retry_count"] || -1
+    end
+  
+    def is_retry(job)
+      current_retries(job) >= 0
+    end
   end
 end

data/lib/sidekiq/instrument/mixin.rb

@@ -12,6 +12,13 @@ module Sidekiq::Instrument
       { tags: ["queue:#{queue_name(worker)}", "worker:#{underscore(class_name(worker))}"] }
     end
 
+    def max_retries(worker)
+      retries = worker.class.get_sidekiq_options['retry'] || Sidekiq[:max_retries]
+      return Sidekiq[:max_retries] if retries.to_s.eql?("true")
+      return 0 if retries.eql?("false")
+      retries
+    end
+
     private
 
     def queue_name(worker)

data/lib/sidekiq/instrument/version.rb

@@ -1,5 +1,5 @@
 module Sidekiq
   module Instrument
-    VERSION = '0.6.1'
+    VERSION = '0.7.0'
   end
 end

data/lib/sidekiq/instrument/worker.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'sidekiq'
 require 'sidekiq/api'
 
@@ -13,7 +15,7 @@ module Sidekiq::Instrument
       workers_size: :workers,
       enqueued: :pending,
       failed: nil
-    }
+    }.freeze
 
     def perform
       info = Sidekiq::Stats.new
@@ -64,8 +66,8 @@ module Sidekiq::Instrument
       return unless WorkerMetrics.enabled
 
       WorkerMetrics.workers_in_queue.each do |key, value|
-        Statter.statsd.gauge("shared.sidekiq.worker_metrics.inqueue.#{key}", value)
-        Statter.dogstatsd&.gauge("shared.sidekiq.worker_metrics.inqueue", value, tags: ["worker:#{key}"])
+        Statter.statsd.gauge("shared.sidekiq.worker_metrics.in_queue.#{key}", value)
+        Statter.dogstatsd&.gauge('shared.sidekiq.worker_metrics.in_queue', value, tags: ["worker:#{key}"])
       end
     end
   end

data/lib/sidekiq/instrument/worker_metrics.rb

@@ -10,7 +10,7 @@ module Sidekiq
       class_attribute :enabled, :namespace
 
       class << self
-        def trace_workers_increment_counter(klass_name, sidekiq_redis_pool_user)
+        def trace_workers_increment_counter(klass_name)
           return unless enabled?
 
           Sidekiq.redis do |redis|

data/sidekiq-instrument.gemspec

@@ -25,6 +25,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'bundler', '~> 2.0', '>= 2.0.2'
   spec.add_development_dependency 'rake', '~> 12.0'
   spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'rubocop', '~> 1.0'
   spec.add_development_dependency 'pry-byebug', '~> 3.4'
   spec.add_development_dependency 'simplecov'
   spec.add_development_dependency 'simplecov-cobertura'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-instrument
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 0.7.0
 platform: ruby
 authors:
 - Loan Application Services
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-05-12 00:00:00.000000000 Z
+date: 2024-08-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -140,6 +140,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: pry-byebug
   requirement: !ruby/object:Gem::Requirement
@@ -182,7 +196,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description:
+description: 
 email:
 - application_services@enova.com
 executables: []
@@ -213,7 +227,7 @@ homepage: https://github.com/enova/sidekiq-instrument
 licenses:
 - MIT
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -228,8 +242,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key:
+rubygems_version: 3.0.3.1
+signing_key: 
 specification_version: 4
 summary: StatsD & DogStatsD Instrumentation for Sidekiq
 test_files: []