sidekiq_utils 1.0.3 → 1.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ab8e9d3a88cd0aab0b51bc4ef8a30cb2524c2a97
-  data.tar.gz: a61dc62b16fc3e057553bd643d9a31c6206e1206
+  metadata.gz: 1dc8efb624d5511e606a14b158b677906be6ef30
+  data.tar.gz: ecf07e503bfa7751c19528ab912dd45536fcbbfb
 SHA512:
-  metadata.gz: ea2fbdcc90c514610236d08ce9dca73546cf7779d83c668843939af279240b6274865e5cb2939f98e1093f8a1c8fed0776b7d1cdeaf47644e80126396478b092
-  data.tar.gz: 3ad39b02be6c048606a285b6fd5f774c46e74a824163cb2404108592295046f1bd5382bcb95f6f9c37910bfcd043feaa22c3b87ddf6084a2af7c16362384fba8
+  metadata.gz: d5ff8cbcf128b4b897836b25ce393863f36e18ae4028fb78e86f5d17fe463019ad02787e65ebc59ae2647e716297be8eb6173fbd5448d900b98c18ec11d5ae23
+  data.tar.gz: 9a1839187a3e53ab5df6944262dac71d44cc4a33a576a86c8ff4b27a4e727f586fb25d00df2bdb8d6f6a145f10c14554edf8e797d6786ee0eedfb48ae04ee70f

data/.circleci/config.yml

@@ -0,0 +1,37 @@
+# Ruby CircleCI 2.0 configuration file
+#
+# Check https://circleci.com/docs/2.0/language-ruby/ for more details
+#
+version: 2
+jobs:
+  build:
+    docker:
+      # specify the version you desire here
+       - image: circleci/ruby:2.2
+       - image: redis:latest
+      
+    working_directory: ~/sidekiq_utils
+
+    steps:
+      - checkout
+
+      # Download and cache dependencies
+      - restore_cache:
+          keys:
+          - v1-dependencies-{{ checksum "Gemfile.lock" }}
+          # fallback to using the latest cache if no exact match is found
+          - v1-dependencies-
+
+      - run:
+          name: install dependencies
+          command: |
+            bundle install --path vendor/bundle
+
+      - save_cache:
+          paths:
+            - ./vendor/bundle
+          key: v1-dependencies-{{ checksum "Gemfile.lock" }}
+        
+      - run:
+          name: rspec spec
+          command: rspec spec

data/.rspec

@@ -0,0 +1 @@
+--require spec_helper

data/Gemfile

@@ -14,3 +14,7 @@ group :development do
   gem "juwelier", "~> 2.1.0"
   gem "simplecov", ">= 0"
 end
+
+group :test do
+  gem 'rspec', '~> 3.7'
+end

data/Gemfile.lock

@@ -13,6 +13,7 @@ GEM
     connection_pool (2.2.1)
     descendants_tracker (0.0.4)
       thread_safe (~> 0.3, >= 0.3.1)
+    diff-lcs (1.3)
     docile (1.1.5)
     faraday (0.12.2)
       multipart-post (>= 1.2, < 3)
@@ -60,6 +61,19 @@ GEM
     rdoc (3.12.2)
       json (~> 1.4)
     redis (4.0.1)
+    rspec (3.7.0)
+      rspec-core (~> 3.7.0)
+      rspec-expectations (~> 3.7.0)
+      rspec-mocks (~> 3.7.0)
+    rspec-core (3.7.0)
+      rspec-support (~> 3.7.0)
+    rspec-expectations (3.7.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.7.0)
+    rspec-mocks (3.7.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.7.0)
+    rspec-support (3.7.0)
     semver (1.0.1)
     shoulda (3.5.0)
       shoulda-context (~> 1.0, >= 1.0.1)
@@ -88,6 +102,7 @@ DEPENDENCIES
   bundler (~> 1.0)
   juwelier (~> 2.1.0)
   rdoc (~> 3.12)
+  rspec (~> 3.7)
   shoulda
   sidekiq (>= 4.0.0)
   simplecov

data/README.md

@@ -119,31 +119,13 @@ end
 ```
 class SolrIndexWorker
   include Sidekiq::Worker
-  
+
   def perform(user_id)
     user = find_optional(User, user_id)
   end
 end
 ```
 
-## Job counter
-
-This will keep a running count of jobs per queue and worker class so that you can inspect what is clogging up high-volume queues without having to iterate over the entire queue.
-
-### Configuration
-
-```
-SidekiqUtils::JobCounter.hook_sidekiq!
-Sidekiq::Web.register SidekiqUtils::WebExtensions::JobCounter
-Sidekiq::Web.tabs["Job counts"] = "job_counts"
-```
-
-### Usage
-
-This will add a "Job counts" tab to your Sidekiq admin which will display the current job count.
-
-Please note that the accuracy of these numbers is not guaranteed as it is non-trivial to keep a running count. However, this is a useful tool when you are inspecting a very long queue.
-
 ## Latency monitor
 
 This will monitor queue latency and report to Slack channels if the latency exceeds the configured threshold.

data/VERSION

@@ -1 +1 @@
-1.0.3
+1.0.5

data/lib/sidekiq_utils.rb

@@ -1,14 +1,11 @@
 require 'sidekiq_utils/middleware/client/additional_serialization'
 require 'sidekiq_utils/middleware/client/deprioritize'
-require 'sidekiq_utils/middleware/client/job_counter'
 
 require 'sidekiq_utils/middleware/server/additional_serialization'
 require 'sidekiq_utils/middleware/server/find_optional'
-require 'sidekiq_utils/middleware/server/job_counter'
 require 'sidekiq_utils/middleware/server/memory_monitor'
 require 'sidekiq_utils/middleware/server/throughput_monitor'
 
-require 'sidekiq_utils/web_extensions/job_counter'
 require 'sidekiq_utils/web_extensions/memory_monitor'
 require 'sidekiq_utils/web_extensions/throughput_monitor'
 
@@ -17,5 +14,4 @@ require 'sidekiq_utils/additional_serialization'
 require 'sidekiq_utils/deprioritize'
 require 'sidekiq_utils/enqueued_jobs_helper'
 require 'sidekiq_utils/find_optional'
-require 'sidekiq_utils/job_counter'
 require 'sidekiq_utils/latency_alert'

data/sidekiq_utils.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: sidekiq_utils 1.0.3 ruby lib
+# stub: sidekiq_utils 1.0.5 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "sidekiq_utils".freeze
-  s.version = "1.0.3"
+  s.version = "1.0.5"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Magnus von Koeller".freeze]
-  s.date = "2017-12-12"
+  s.date = "2019-01-23"
   s.description = "Tools that make working with a major Sidekiq installation more fun.".freeze
   s.email = "magnus@angel.co".freeze
   s.extra_rdoc_files = [
@@ -19,6 +19,8 @@ Gem::Specification.new do |s|
     "README.md"
   ]
   s.files = [
+    ".circleci/config.yml",
+    ".rspec",
     "Gemfile",
     "Gemfile.lock",
     "LICENSE",
@@ -30,28 +32,26 @@ Gem::Specification.new do |s|
     "lib/sidekiq_utils/deprioritize.rb",
     "lib/sidekiq_utils/enqueued_jobs_helper.rb",
     "lib/sidekiq_utils/find_optional.rb",
-    "lib/sidekiq_utils/job_counter.rb",
     "lib/sidekiq_utils/latency_alert.rb",
+    "lib/sidekiq_utils/middleware/.DS_Store",
     "lib/sidekiq_utils/middleware/client/additional_serialization.rb",
     "lib/sidekiq_utils/middleware/client/deprioritize.rb",
-    "lib/sidekiq_utils/middleware/client/job_counter.rb",
     "lib/sidekiq_utils/middleware/server/additional_serialization.rb",
     "lib/sidekiq_utils/middleware/server/find_optional.rb",
-    "lib/sidekiq_utils/middleware/server/job_counter.rb",
     "lib/sidekiq_utils/middleware/server/memory_monitor.rb",
     "lib/sidekiq_utils/middleware/server/throughput_monitor.rb",
     "lib/sidekiq_utils/redis_monitor_storage.rb",
-    "lib/sidekiq_utils/web_extensions/job_counter.rb",
     "lib/sidekiq_utils/web_extensions/memory_monitor.rb",
     "lib/sidekiq_utils/web_extensions/throughput_monitor.rb",
     "lib/sidekiq_utils/web_extensions/views/job_counts.erb",
     "lib/sidekiq_utils/web_extensions/views/memory.erb",
     "lib/sidekiq_utils/web_extensions/views/throughput.erb",
-    "sidekiq_utils.gemspec"
+    "sidekiq_utils.gemspec",
+    "spec/spec_helper.rb"
   ]
   s.homepage = "http://github.com/venturehacks/sidekiq_angels".freeze
   s.licenses = ["MIT".freeze]
-  s.rubygems_version = "2.6.13".freeze
+  s.rubygems_version = "2.6.14.3".freeze
   s.summary = "Tools that make working with a major Sidekiq installation more fun.".freeze
 
   if s.respond_to? :specification_version then

data/spec/spec_helper.rb

@@ -0,0 +1,102 @@
+ENV['RAILS_ENV'] ||= 'test'
+
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# The generated `.rspec` file contains `--require spec_helper` which will cause
+# this file to always be loaded, without a need to explicitly require it in any
+# files.
+#
+# Given that it is always loaded, you are encouraged to keep this file as
+# light-weight as possible. Requiring heavyweight dependencies from this file
+# will add to the boot time of your test suite on EVERY test run, even for an
+# individual file that may not need all of that loaded. Instead, consider making
+# a separate helper file that requires the additional dependencies and performs
+# the additional setup, and require it from the spec files that actually need
+# it.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  # rspec-expectations config goes here. You can use an alternate
+  # assertion/expectation library such as wrong or the stdlib/minitest
+  # assertions if you prefer.
+  config.expect_with :rspec do |expectations|
+    # This option will default to `true` in RSpec 4. It makes the `description`
+    # and `failure_message` of custom matchers include text for helper methods
+    # defined using `chain`, e.g.:
+    #     be_bigger_than(2).and_smaller_than(4).description
+    #     # => "be bigger than 2 and smaller than 4"
+    # ...rather than:
+    #     # => "be bigger than 2"
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  # rspec-mocks config goes here. You can use an alternate test double
+  # library (such as bogus or mocha) by changing the `mock_with` option here.
+  config.mock_with :rspec do |mocks|
+    # Prevents you from mocking or stubbing a method that does not exist on
+    # a real object. This is generally recommended, and will default to
+    # `true` in RSpec 4.
+    mocks.verify_partial_doubles = true
+  end
+
+  # This option will default to `:apply_to_host_groups` in RSpec 4 (and will
+  # have no way to turn it off -- the option exists only for backwards
+  # compatibility in RSpec 3). It causes shared context metadata to be
+  # inherited by the metadata hash of host groups and examples, rather than
+  # triggering implicit auto-inclusion in groups with matching metadata.
+  config.shared_context_metadata_behavior = :apply_to_host_groups
+
+# The settings below are suggested to provide a good initial experience
+# with RSpec, but feel free to customize to your heart's content.
+=begin
+  # This allows you to limit a spec run to individual examples or groups
+  # you care about by tagging them with `:focus` metadata. When nothing
+  # is tagged with `:focus`, all examples get run. RSpec also provides
+  # aliases for `it`, `describe`, and `context` that include `:focus`
+  # metadata: `fit`, `fdescribe` and `fcontext`, respectively.
+  config.filter_run_when_matching :focus
+
+  # Allows RSpec to persist some state between runs in order to support
+  # the `--only-failures` and `--next-failure` CLI options. We recommend
+  # you configure your source control system to ignore this file.
+  config.example_status_persistence_file_path = "spec/examples.txt"
+
+  # Limits the available syntax to the non-monkey patched syntax that is
+  # recommended. For more details, see:
+  #   - http://rspec.info/blog/2012/06/rspecs-new-expectation-syntax/
+  #   - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
+  #   - http://rspec.info/blog/2014/05/notable-changes-in-rspec-3/#zero-monkey-patching-mode
+  config.disable_monkey_patching!
+
+  # This setting enables warnings. It's recommended, but in some cases may
+  # be too noisy due to issues in dependencies.
+  config.warnings = true
+
+  # Many RSpec users commonly either run the entire suite or an individual
+  # file, and it's useful to allow more verbose output when running an
+  # individual spec file.
+  if config.files_to_run.one?
+    # Use the documentation formatter for detailed output,
+    # unless a formatter has already been configured
+    # (e.g. via a command-line flag).
+    config.default_formatter = "doc"
+  end
+
+  # Print the 10 slowest examples and example groups at the
+  # end of the spec run, to help surface which specs are running
+  # particularly slow.
+  config.profile_examples = 10
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = :random
+
+  # Seed global randomization in this process using the `--seed` CLI option.
+  # Setting this allows you to use `--seed` to deterministically reproduce
+  # test failures related to randomization by passing the same `--seed` value
+  # as the one that triggered the failure.
+  Kernel.srand config.seed
+=end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq_utils
 version: !ruby/object:Gem::Version
-  version: 1.0.3
+  version: 1.0.5
 platform: ruby
 authors:
 - Magnus von Koeller
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-12-12 00:00:00.000000000 Z
+date: 2019-01-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -102,6 +102,8 @@ extra_rdoc_files:
 - LICENSE
 - README.md
 files:
+- ".circleci/config.yml"
+- ".rspec"
 - Gemfile
 - Gemfile.lock
 - LICENSE
@@ -113,24 +115,22 @@ files:
 - lib/sidekiq_utils/deprioritize.rb
 - lib/sidekiq_utils/enqueued_jobs_helper.rb
 - lib/sidekiq_utils/find_optional.rb
-- lib/sidekiq_utils/job_counter.rb
 - lib/sidekiq_utils/latency_alert.rb
+- lib/sidekiq_utils/middleware/.DS_Store
 - lib/sidekiq_utils/middleware/client/additional_serialization.rb
 - lib/sidekiq_utils/middleware/client/deprioritize.rb
-- lib/sidekiq_utils/middleware/client/job_counter.rb
 - lib/sidekiq_utils/middleware/server/additional_serialization.rb
 - lib/sidekiq_utils/middleware/server/find_optional.rb
-- lib/sidekiq_utils/middleware/server/job_counter.rb
 - lib/sidekiq_utils/middleware/server/memory_monitor.rb
 - lib/sidekiq_utils/middleware/server/throughput_monitor.rb
 - lib/sidekiq_utils/redis_monitor_storage.rb
-- lib/sidekiq_utils/web_extensions/job_counter.rb
 - lib/sidekiq_utils/web_extensions/memory_monitor.rb
 - lib/sidekiq_utils/web_extensions/throughput_monitor.rb
 - lib/sidekiq_utils/web_extensions/views/job_counts.erb
 - lib/sidekiq_utils/web_extensions/views/memory.erb
 - lib/sidekiq_utils/web_extensions/views/throughput.erb
 - sidekiq_utils.gemspec
+- spec/spec_helper.rb
 homepage: http://github.com/venturehacks/sidekiq_angels
 licenses:
 - MIT
@@ -151,7 +151,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.13
+rubygems_version: 2.6.14.3
 signing_key: 
 specification_version: 4
 summary: Tools that make working with a major Sidekiq installation more fun.

data/lib/sidekiq_utils/job_counter.rb

@@ -1,161 +0,0 @@
-module SidekiqUtils
-  module JobCounter
-    REDIS_KEY = 'sidekiq_utils_job_counter'
-    LOCK = Mutex.new
-    SYNC_COUNTS_EVERY = 1.second
-
-    class << self
-      def increment(job)
-        change_count(job, 1)
-      end
-
-      def decrement(job)
-        change_count(job, -1)
-      end
-
-      def counts
-        Sidekiq.redis do |redis|
-          redis.hgetall(REDIS_KEY).each_with_object({}) do |(key, count), ret_hash|
-            key_hash = JSON.parse(key, symbolize_names: true)
-
-            count = count.to_i
-            if count != 0
-              ret_hash[key_hash[:queue]] ||= {}
-              ret_hash[key_hash[:queue]][key_hash[:job]] = count
-            end
-          end
-        end
-      end
-
-      def reset!
-        Sidekiq.redis do |redis|
-          redis.del(REDIS_KEY)
-        end
-      end
-
-      def hook_sidekiq!
-        Sidekiq::SortedEntry.prepend(SortedEntry::JobCounterExtension)
-        Sidekiq::ScheduledSet.prepend(ScheduledSet::JobCounterExtension)
-        Sidekiq::Job.prepend(Job::JobCounterExtension)
-        Sidekiq::Queue.prepend(Queue::JobCounterExtension)
-        Sidekiq::JobSet.prepend(JobSet::JobCounterExtension)
-      end
-
-      private
-      def change_count(job, change_by)
-        unless [-1, 1].include?(change_by)
-          fail("Unsupported change_by value: #{change_by}")
-        end
-
-        job_key = SidekiqUtils::RedisMonitorStorage.job_prefix(
-            job, unwrap_arguments: true)
-        hash_key = {
-          queue: job['queue'],
-          job: job_key,
-        }.to_json
-
-        LOCK.synchronize do
-          @counts_to_flush ||= {}
-          @counts_to_flush[hash_key] ||= 0
-          @counts_to_flush[hash_key] += change_by
-
-          if !Rails.env.test?
-            @sync_thread ||= Thread.new do
-              sleep SYNC_COUNTS_EVERY
-              sync_to_redis
-            end
-          end
-        end
-
-        sync_to_redis if Rails.env.test?
-      end
-
-      def sync_to_redis
-        local_counts_to_flush = nil
-        LOCK.synchronize do
-          local_counts_to_flush = @counts_to_flush
-          @counts_to_flush = {}
-          @sync_thread = nil
-        end
-        (local_counts_to_flush || {}).each do |hash_key, change_by|
-          Sidekiq.redis do |redis|
-            count = redis.hincrby(REDIS_KEY, hash_key, change_by)
-            if count < 0 && change_by < 0
-              # this shouldn't happen, but it could when we first deploy this.
-              # just makes sure we don't end up with negative values here
-              # which don't make sense
-              #
-              # we only ever increment by the same amount we just did because
-              # we can't be responsible for more of a discrepancy at this
-              # point and we don't want multiple threads to overcorrect for
-              # each other
-              count = redis.hincrby(REDIS_KEY, hash_key, -1*change_by)
-            end
-          end
-        end
-      end
-      at_exit { SidekiqUtils::JobCounter.send(:sync_to_redis) }
-    end
-  end
-
-  class SortedEntry
-    module JobCounterExtension
-      def delete
-        if super
-          JobCounter.decrement(item)
-        end
-      end
-
-      private
-
-      def remove_job
-        super do |message|
-          JobCounter.decrement(Sidekiq.load_json(message))
-          yield message
-        end
-      end
-    end
-  end
-
-  class ScheduledSet
-    module JobCounterExtension
-      def delete
-        if super
-          JobCounter.decrement(item)
-        end
-      end
-    end
-  end
-
-  class Job
-    module JobCounterExtension
-      def delete
-        super
-        JobCounter.decrement(item)
-      end
-    end
-  end
-
-  class Queue
-    module JobCounterExtension
-      def clear
-        super
-      end
-    end
-  end
-
-  class JobSet
-    module JobCounterExtension
-      def clear
-        each(&:delete)
-        super
-      end
-
-      def delete_by_value(name, value)
-        if super
-          JobCounter.decrement(Sidekiq.load_json(value))
-        end
-      end
-    end
-  end
-end

data/lib/sidekiq_utils/middleware/client/job_counter.rb

@@ -1,17 +0,0 @@
-module SidekiqUtils
-  module Middleware
-    module Client
-      class JobCounter
-        def call(worker_class, job, queue, redis_pool)
-          unless job['at']
-            # don't count when jobs get put on the scheduled set, because
-            # otherwise we'll double-count them when they get popped and moved
-            # to a work queue.
-            SidekiqUtils::JobCounter.increment(job)
-          end
-          yield
-        end
-      end
-    end
-  end
-end

data/lib/sidekiq_utils/middleware/server/job_counter.rb

@@ -1,15 +0,0 @@
-module SidekiqUtils
-  module Middleware
-    module Server
-      class JobCounter
-        def call(worker, job, queue)
-          # we decrement here whether the job succeeds or not, because
-          # re-enqueuing from the retry queue triggers the client middleware
-          # and thus another increment even in the case of an error
-          SidekiqUtils::JobCounter.decrement(job)
-          yield
-        end
-      end
-    end
-  end
-end

data/lib/sidekiq_utils/web_extensions/job_counter.rb

@@ -1,35 +0,0 @@
-module SidekiqUtils
-  module WebExtensions
-    module JobCounter
-      def self.registered(app)
-        view_path = File.join(File.expand_path("..", __FILE__), "views")
-
-        require 'active_support/number_helper'
-        app.get("/job_counts") do
-          @throughput = SidekiqUtils::RedisMonitorStorage.
-              retrieve('sidekiq_elapsed', 'elapsed')
-          @counts = []
-          SidekiqUtils::JobCounter.counts.each do |queue, job_counts|
-            job_counts.each do |job, count|
-              values = { queue: queue, job: job, count: count }
-              if (values[:avg_runtime] = @throughput[job].try!(:[], 'average'))
-                execution_time = values[:count] * values[:avg_runtime].to_f / 1_000
-                values[:runtime_day] = days =
-                  (execution_time / 1.day).floor
-                values[:runtime_hour] = hours =
-                  ((execution_time - days.days) / 1.hour).floor
-                values[:runtime_min] = (
-                  (execution_time - days.days - hours.hours) / 1.minute
-                ).round
-              end
-              @counts << values
-            end
-          end
-          @counts.sort_by! {|x| -1 * x[:count] }
-
-          render(:erb, File.read(File.join(view_path, "job_counts.erb")))
-        end
-      end
-    end
-  end
-end