yabeda-sidekiq 0.1.3 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 72f1df666e70373b15a7fc54732f14e6701a3fbfdd2a19e5a3703e542c6a13d4
-  data.tar.gz: c38ed88429527e43edff2d445c8940d8bf52babd272fa5b1b7bbb37865b6ecff
+  metadata.gz: 660902666ec22bdc3a244a45bd15ac3a67b8dc2d13cd703c779453e689002b93
+  data.tar.gz: 5b45a914f585959fceceda65d95a72bf6415f9e6a3bb889f54243ac8f887adec
 SHA512:
-  metadata.gz: 1d31f1bf0323344baf0e1a2707bd89661c09f14f664dabd8fa9fff347e308538608c9922d91d4f684b2a34d89ca95218dab7f29bfc0b359bd77057e34205a87a
-  data.tar.gz: 3764a37a9aa8b9ccd715ca049e55af864324fe5927e2496825b1d0f222b63b452dbaa9f8b9e50ce4e29a9b044e5747cc711b4de4f27813a7b4e9796f6419e01a
+  metadata.gz: 22be4dea939719b79f230d80ddcf38269cc476d3cf51c6664938b5bab6e93e2e924dbebcdcbcbd9dde59e0822f28c33a58b83be483015dad9ed32000cb9fceeb
+  data.tar.gz: 3d3c314d83b3943fcbd5d110f44826abbad4beb50c61120f5f26d33c398cc02bf970c36a6272143f3ae27dd9adfc73f42e69f898460461f952519bb8de9a768f

data/.rubocop.yml

@@ -1,17 +1,18 @@
 ---
 require:
-- rubocop-rspec
+  - rubocop-rspec
 
 AllCops:
   TargetRubyVersion: 2.3
 
 Metrics/BlockLength:
+  Enabled: false
   Exclude:
   - "Gemfile"
   - "spec/**/*"
 
-Style/BracesAroundHashParameters:
-  EnforcedStyle: context_dependent
+Layout/LineLength:
+  Max: 160
 
 Style/StringLiterals:
   EnforcedStyle: double_quotes
@@ -44,3 +45,23 @@ Style/TrailingCommaInHashLiteral:
   Enabled: true
   EnforcedStyleForMultiline: consistent_comma
 
+Style/HashEachMethods:
+  Enabled: true
+
+Style/HashTransformKeys:
+  Enabled: true
+
+Style/HashTransformValues:
+  Enabled: true
+
+RSpec/ExampleLength:
+  Enabled: false
+
+Style/Documentation:
+  Enabled: false
+
+Metrics/MethodLength:
+  Max: 15
+
+Metrics/AbcSize:
+  Max: 17

data/.travis.yml

@@ -1,5 +1,7 @@
 sudo: false
 language: ruby
 rvm:
-  - 2.5.1
-before_install: gem install bundler -v 1.16.1
+  - 2.7.1
+  - 2.6.6
+  - 2.5.8
+before_install: gem install bundler -v '~> 2.0'

data/CHANGELOG.md

@@ -0,0 +1,65 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
+and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
+
+## 0.7.0 - 2020-07-15
+
+### Changed
+
+ - Tags from `yabeda_tags` method are applied to all metrics collected inside a job, not only sidekiq-specific. See [#14](https://github.com/yabeda-rb/yabeda-sidekiq/issues/14). @Envek
+
+## 0.6.0 - 2020-07-15
+
+### Added
+
+ - Ability to override or add tags for every job via `yabeda_tags` method. @Envek
+
+## 0.5.0 - 2020-02-20
+
+### Added
+
+ - New `sidekiq_job_latency` histogram to track latency statistics of different job classes. [#9](https://github.com/yabeda-rb/yabeda-sidekiq/pull/9) by [@asusikov]
+
+### Changed
+
+ - **BREAKING CHANGE!** Renamed `sidekiq_jobs_latency` gauge to `sidekiq_queue_latency` to better describe its purpose and differentiate with the new histogram. [#9](https://github.com/yabeda-rb/yabeda-sidekiq/pull/9) by [@asusikov]
+
+## 0.2.0 - 2020-01-14
+
+### Changed
+
+ - Added `tags` option to metric declarations for compatibility with yabeda and yabeda-prometheus 0.2. @Envek
+
+## 0.1.4 - 2019-10-07
+
+### Added
+
+ - Require of core yabeda gem [#4](https://github.com/yabeda-rb/yabeda-sidekiq/pull/4). [@dsalahutdinov]
+
+## 0.1.3 - 2018-10-25
+
+### Fixed
+
+ - Require of core yabeda gem [#1](https://github.com/yabeda-rb/yabeda-sidekiq/issues/1). @Envek
+
+## 0.1.2 - 2018-10-17
+
+### Changed
+
+ - Renamed evil-metrics-sidekiq gem to yabeda-sidekiq. @Envek
+
+## 0.1.1 - 2018-10-05
+
+### Changed
+
+ - Automatic add client and server middlewares to Sidekiq. @Envek
+
+## 0.1.0 - 2018-10-03
+
+ - Initial release of evil-metrics-sidekiq gem. @Envek
+
+[@dsalahutdinov]: https://github.com/dsalahutdinov "Salahutdinov Dmitry"
+[@asusikov]: https://github.com/asusikov "Alexander Susikov"

data/Gemfile

@@ -11,6 +11,6 @@ group :development, :test do
   gem "pry"
   gem "pry-byebug", platform: :mri
 
-  gem "rubocop"
+  gem "rubocop", "~> 0.80.0"
   gem "rubocop-rspec"
 end

data/README.md

@@ -2,10 +2,12 @@
 
 Built-in metrics for [Sidekiq] monitoring out of the box! Part of the [yabeda] suite.
 
+Sample Grafana dashboard ID: [11667](https://grafana.com/grafana/dashboards/11667)
+
 ## Installation
 
 ```ruby
-gem 'yabeda-rails'
+gem 'yabeda-sidekiq'
 # Then add monitoring system adapter, e.g.:
 # gem 'yabeda-prometheus'
 ```
@@ -14,6 +16,14 @@ And then execute:
 
     $ bundle
 
+If you're not on Rails then configure Yabeda after your application was initialized:
+
+```ruby
+Yabeda.configure!
+```
+
+_If you're using Ruby on Rails then Yabeda will configure itself automatically!_
+
 **And that is it!** Sidekiq metrics are being collected!
 
 Additionally, depending on your adapter, you may want to setup metrics export. E.g. for [yabeda-prometheus]:
@@ -31,6 +41,8 @@ end
  - Number of jobs have been finished successfully: `sidekiq_jobs_success_total` (segmented by queue and class name)
  - Number of jobs have been failed: `sidekiq_jobs_failed_total` (segmented by queue and class name)
  - Time of job run: `sidekiq_job_runtime` (seconds per job execution, segmented by queue and class name)
+ - Time of the queue latency `sidekiq_queue_latency` (the difference in seconds since the oldest job in the queue was enqueued)
+ - Time of the job latency `sidekiq_job_latency` (the difference in seconds since the enqueuing until running job)
  - Number of jobs in queues: `sidekiq_jobs_waiting_count` (segmented by queue)
  - Number of scheduled jobs:`sidekiq_jobs_scheduled_count`
  - Number of jobs in retry set: `sidekiq_jobs_retry_count`
@@ -38,6 +50,29 @@ end
  - Active workers count: `sidekiq_active_processes`
  - Active processes count: `sidekiq_active_workers_count`
 
+## Custom tags
+
+You can add additional tags to these metrics by declaring `yabeda_tags` method in your worker.
+
+```ruby
+# This block is optional but some adapters (like Prometheus) requires that all tags should be declared in advance
+Yabeda.configure do
+  default_tag :importance, nil
+end
+
+class MyWorker
+  include Sidekiq::Worker
+
+  def yabeda_tags(*params) # This method will be called first, before +perform+
+    { importance: extract_importance(params) }
+  end
+
+  def perform(*params)
+    # Your logic here
+  end
+end
+```
+
 # Roadmap (TODO or Help wanted)
 
  - Implement optional segmentation of retry/schedule/dead sets

data/Rakefile

@@ -2,7 +2,10 @@
 
 require "bundler/gem_tasks"
 require "rspec/core/rake_task"
+require "rubocop/rake_task"
+
+RuboCop::RakeTask.new
 
 RSpec::Core::RakeTask.new(:spec)
 
-task default: :spec
+task default: %i[rubocop spec]

data/lib/yabeda/sidekiq.rb

@@ -12,26 +12,35 @@ module Yabeda
   module Sidekiq
     LONG_RUNNING_JOB_RUNTIME_BUCKETS = [
       0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, # standard (from Prometheus)
-      30, 60, 120, 300, 1800, 3600, 21_600 # Sidekiq tasks may be very long-running
+      30, 60, 120, 300, 1800, 3600, 21_600, # Sidekiq tasks may be very long-running
     ].freeze
 
     Yabeda.configure do
       group :sidekiq
 
-      counter :jobs_enqueued_total, comment: "A counter of the total number of jobs sidekiq enqueued."
+      counter :jobs_enqueued_total, tags: %i[queue worker], comment: "A counter of the total number of jobs sidekiq enqueued."
 
       next unless ::Sidekiq.server?
 
-      counter   :jobs_executed_total,  comment: "A counter of the total number of jobs sidekiq executed."
-      counter   :jobs_success_total,   comment: "A counter of the total number of jobs successfully processed by sidekiq."
-      counter   :jobs_failed_total,    comment: "A counter of the total number of jobs failed in sidekiq."
-      gauge     :jobs_waiting_count,   comment: "The number of jobs waiting to process in sidekiq."
-      gauge     :active_workers_count, comment: "The number of currently running machines with sidekiq workers."
-      gauge     :jobs_scheduled_count, comment: "The number of jobs scheduled for later execution."
-      gauge     :jobs_retry_count,     comment: "The number of failed jobs waiting to be retried"
-      gauge     :jobs_dead_count,      comment: "The number of jobs exceeded their retry count."
-      gauge     :active_processes,     comment: "The number of active Sidekiq worker processes."
-      histogram :job_runtime, unit: :seconds, per: :job, comment: "A histogram of the job execution time.",
+      counter   :jobs_executed_total,  tags: %i[queue worker], comment: "A counter of the total number of jobs sidekiq executed."
+      counter   :jobs_success_total,   tags: %i[queue worker], comment: "A counter of the total number of jobs successfully processed by sidekiq."
+      counter   :jobs_failed_total,    tags: %i[queue worker], comment: "A counter of the total number of jobs failed in sidekiq."
+
+      gauge     :jobs_waiting_count,   tags: %i[queue], comment: "The number of jobs waiting to process in sidekiq."
+      gauge     :active_workers_count, tags: [],        comment: "The number of currently running machines with sidekiq workers."
+      gauge     :jobs_scheduled_count, tags: [],        comment: "The number of jobs scheduled for later execution."
+      gauge     :jobs_retry_count,     tags: [],        comment: "The number of failed jobs waiting to be retried"
+      gauge     :jobs_dead_count,      tags: [],        comment: "The number of jobs exceeded their retry count."
+      gauge     :active_processes,     tags: [],        comment: "The number of active Sidekiq worker processes."
+      gauge     :queue_latency,        tags: %i[queue], comment: "The queue latency, the difference in seconds since the oldest job in the queue was enqueued"
+
+      histogram :job_latency, comment: "The job latency, the difference in seconds between enqueued and running time",
+                              unit: :seconds, per: :job,
+                              tags: %i[queue worker],
+                              buckets: LONG_RUNNING_JOB_RUNTIME_BUCKETS
+      histogram :job_runtime, comment: "A histogram of the job execution time.",
+                              unit: :seconds, per: :job,
+                              tags: %i[queue worker],
                               buckets: LONG_RUNNING_JOB_RUNTIME_BUCKETS
 
       collect do
@@ -46,6 +55,10 @@ module Yabeda
         sidekiq_active_processes.set({}, stats.processes_size)
         sidekiq_jobs_retry_count.set({}, stats.retry_size)
 
+        ::Sidekiq::Queue.all.each do |queue|
+          sidekiq_queue_latency.set({ queue: queue.name }, queue.latency)
+        end
+
         # That is quite slow if your retry set is large
         # I don't want to enable it by default
         # retries_by_queues =
@@ -80,9 +93,17 @@ module Yabeda
 
       def worker_class(worker, job)
         if defined?(ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper)
-          return job["wrapped"] if worker.is_a?(ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper)
+          if worker.is_a?(ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper) || worker == ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper
+            return job["wrapped"].to_s
+          end
         end
-        (worker.is_a?(String) ? worker : worker.class).to_s
+        (worker.is_a?(String) || worker.is_a?(Class) ? worker : worker.class).to_s
+      end
+
+      def custom_tags(worker, job)
+        return {} unless worker.respond_to?(:yabeda_tags)
+
+        worker.method(:yabeda_tags).arity.zero? ? worker.yabeda_tags : worker.yabeda_tags(*job["args"])
       end
     end
   end

data/lib/yabeda/sidekiq/server_middleware.rb

@@ -4,11 +4,17 @@ module Yabeda
   module Sidekiq
     # Sidekiq worker middleware
     class ServerMiddleware
+      # rubocop: disable Metrics/AbcSize, Metrics/MethodLength:
       def call(worker, job, queue)
-        labels = Yabeda::Sidekiq.labelize(worker, job, queue)
-        start = Time.now
+        custom_tags = Yabeda::Sidekiq.custom_tags(worker, job).to_h
+        labels = Yabeda::Sidekiq.labelize(worker, job, queue).merge(custom_tags)
+        start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
         begin
-          yield
+          job_instance = ::Sidekiq::Job.new(job)
+          Yabeda.sidekiq_job_latency.measure(labels, job_instance.latency)
+          Yabeda.with_tags(**custom_tags) do
+            yield
+          end
           Yabeda.sidekiq_jobs_success_total.increment(labels)
         rescue Exception # rubocop: disable Lint/RescueException
           Yabeda.sidekiq_jobs_failed_total.increment(labels)
@@ -18,11 +24,12 @@ module Yabeda
           Yabeda.sidekiq_jobs_executed_total.increment(labels)
         end
       end
+      # rubocop: enable Metrics/AbcSize, Metrics/MethodLength:
 
       private
 
       def elapsed(start)
-        (Time.now - start).round(3)
+        (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start).round(3)
       end
     end
   end

data/lib/yabeda/sidekiq/version.rb

@@ -2,6 +2,6 @@
 
 module Yabeda
   module Sidekiq
-    VERSION = "0.1.3"
+    VERSION = "0.7.0"
   end
 end

data/yabeda-sidekiq.gemspec

@@ -22,10 +22,11 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "yabeda"
   spec.add_dependency "sidekiq"
+  spec.add_dependency "yabeda", "~> 0.6"
 
-  spec.add_development_dependency "bundler", "~> 1.16"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "activejob", ">= 6.0"
+  spec.add_development_dependency "bundler", "~> 2.0"
+  spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.0"
 end

metadata

@@ -1,17 +1,17 @@
 --- !ruby/object:Gem::Specification
 name: yabeda-sidekiq
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.7.0
 platform: ruby
 authors:
 - Andrey Novikov
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-10-25 00:00:00.000000000 Z
+date: 2020-08-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: yabeda
+  name: sidekiq
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -25,47 +25,61 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: sidekiq
+  name: yabeda
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: activejob
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '6.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6.0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -92,6 +106,7 @@ files:
 - ".rspec"
 - ".rubocop.yml"
 - ".travis.yml"
+- CHANGELOG.md
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -122,8 +137,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Extensible Prometheus exporter for monitoring your Sidekiq