sidekiq_utils 1.0.3 → 1.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.circleci/config.yml +37 -0
- data/.rspec +1 -0
- data/Gemfile +4 -0
- data/Gemfile.lock +15 -0
- data/README.md +1 -19
- data/VERSION +1 -1
- data/lib/sidekiq_utils.rb +0 -4
- data/lib/sidekiq_utils/middleware/.DS_Store +0 -0
- data/sidekiq_utils.gemspec +9 -9
- data/spec/spec_helper.rb +102 -0
- metadata +7 -7
- data/lib/sidekiq_utils/job_counter.rb +0 -161
- data/lib/sidekiq_utils/middleware/client/job_counter.rb +0 -17
- data/lib/sidekiq_utils/middleware/server/job_counter.rb +0 -15
- data/lib/sidekiq_utils/web_extensions/job_counter.rb +0 -35
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1dc8efb624d5511e606a14b158b677906be6ef30
|
4
|
+
data.tar.gz: ecf07e503bfa7751c19528ab912dd45536fcbbfb
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d5ff8cbcf128b4b897836b25ce393863f36e18ae4028fb78e86f5d17fe463019ad02787e65ebc59ae2647e716297be8eb6173fbd5448d900b98c18ec11d5ae23
|
7
|
+
data.tar.gz: 9a1839187a3e53ab5df6944262dac71d44cc4a33a576a86c8ff4b27a4e727f586fb25d00df2bdb8d6f6a145f10c14554edf8e797d6786ee0eedfb48ae04ee70f
|
@@ -0,0 +1,37 @@
|
|
1
|
+
# Ruby CircleCI 2.0 configuration file
|
2
|
+
#
|
3
|
+
# Check https://circleci.com/docs/2.0/language-ruby/ for more details
|
4
|
+
#
|
5
|
+
version: 2
|
6
|
+
jobs:
|
7
|
+
build:
|
8
|
+
docker:
|
9
|
+
# specify the version you desire here
|
10
|
+
- image: circleci/ruby:2.2
|
11
|
+
- image: redis:latest
|
12
|
+
|
13
|
+
working_directory: ~/sidekiq_utils
|
14
|
+
|
15
|
+
steps:
|
16
|
+
- checkout
|
17
|
+
|
18
|
+
# Download and cache dependencies
|
19
|
+
- restore_cache:
|
20
|
+
keys:
|
21
|
+
- v1-dependencies-{{ checksum "Gemfile.lock" }}
|
22
|
+
# fallback to using the latest cache if no exact match is found
|
23
|
+
- v1-dependencies-
|
24
|
+
|
25
|
+
- run:
|
26
|
+
name: install dependencies
|
27
|
+
command: |
|
28
|
+
bundle install --path vendor/bundle
|
29
|
+
|
30
|
+
- save_cache:
|
31
|
+
paths:
|
32
|
+
- ./vendor/bundle
|
33
|
+
key: v1-dependencies-{{ checksum "Gemfile.lock" }}
|
34
|
+
|
35
|
+
- run:
|
36
|
+
name: rspec spec
|
37
|
+
command: rspec spec
|
data/.rspec
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
--require spec_helper
|
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
@@ -13,6 +13,7 @@ GEM
|
|
13
13
|
connection_pool (2.2.1)
|
14
14
|
descendants_tracker (0.0.4)
|
15
15
|
thread_safe (~> 0.3, >= 0.3.1)
|
16
|
+
diff-lcs (1.3)
|
16
17
|
docile (1.1.5)
|
17
18
|
faraday (0.12.2)
|
18
19
|
multipart-post (>= 1.2, < 3)
|
@@ -60,6 +61,19 @@ GEM
|
|
60
61
|
rdoc (3.12.2)
|
61
62
|
json (~> 1.4)
|
62
63
|
redis (4.0.1)
|
64
|
+
rspec (3.7.0)
|
65
|
+
rspec-core (~> 3.7.0)
|
66
|
+
rspec-expectations (~> 3.7.0)
|
67
|
+
rspec-mocks (~> 3.7.0)
|
68
|
+
rspec-core (3.7.0)
|
69
|
+
rspec-support (~> 3.7.0)
|
70
|
+
rspec-expectations (3.7.0)
|
71
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
72
|
+
rspec-support (~> 3.7.0)
|
73
|
+
rspec-mocks (3.7.0)
|
74
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
75
|
+
rspec-support (~> 3.7.0)
|
76
|
+
rspec-support (3.7.0)
|
63
77
|
semver (1.0.1)
|
64
78
|
shoulda (3.5.0)
|
65
79
|
shoulda-context (~> 1.0, >= 1.0.1)
|
@@ -88,6 +102,7 @@ DEPENDENCIES
|
|
88
102
|
bundler (~> 1.0)
|
89
103
|
juwelier (~> 2.1.0)
|
90
104
|
rdoc (~> 3.12)
|
105
|
+
rspec (~> 3.7)
|
91
106
|
shoulda
|
92
107
|
sidekiq (>= 4.0.0)
|
93
108
|
simplecov
|
data/README.md
CHANGED
@@ -119,31 +119,13 @@ end
|
|
119
119
|
```
|
120
120
|
class SolrIndexWorker
|
121
121
|
include Sidekiq::Worker
|
122
|
-
|
122
|
+
|
123
123
|
def perform(user_id)
|
124
124
|
user = find_optional(User, user_id)
|
125
125
|
end
|
126
126
|
end
|
127
127
|
```
|
128
128
|
|
129
|
-
## Job counter
|
130
|
-
|
131
|
-
This will keep a running count of jobs per queue and worker class so that you can inspect what is clogging up high-volume queues without having to iterate over the entire queue.
|
132
|
-
|
133
|
-
### Configuration
|
134
|
-
|
135
|
-
```
|
136
|
-
SidekiqUtils::JobCounter.hook_sidekiq!
|
137
|
-
Sidekiq::Web.register SidekiqUtils::WebExtensions::JobCounter
|
138
|
-
Sidekiq::Web.tabs["Job counts"] = "job_counts"
|
139
|
-
```
|
140
|
-
|
141
|
-
### Usage
|
142
|
-
|
143
|
-
This will add a "Job counts" tab to your Sidekiq admin which will display the current job count.
|
144
|
-
|
145
|
-
Please note that the accuracy of these numbers is not guaranteed as it is non-trivial to keep a running count. However, this is a useful tool when you are inspecting a very long queue.
|
146
|
-
|
147
129
|
## Latency monitor
|
148
130
|
|
149
131
|
This will monitor queue latency and report to Slack channels if the latency exceeds the configured threshold.
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
1.0.
|
1
|
+
1.0.5
|
data/lib/sidekiq_utils.rb
CHANGED
@@ -1,14 +1,11 @@
|
|
1
1
|
require 'sidekiq_utils/middleware/client/additional_serialization'
|
2
2
|
require 'sidekiq_utils/middleware/client/deprioritize'
|
3
|
-
require 'sidekiq_utils/middleware/client/job_counter'
|
4
3
|
|
5
4
|
require 'sidekiq_utils/middleware/server/additional_serialization'
|
6
5
|
require 'sidekiq_utils/middleware/server/find_optional'
|
7
|
-
require 'sidekiq_utils/middleware/server/job_counter'
|
8
6
|
require 'sidekiq_utils/middleware/server/memory_monitor'
|
9
7
|
require 'sidekiq_utils/middleware/server/throughput_monitor'
|
10
8
|
|
11
|
-
require 'sidekiq_utils/web_extensions/job_counter'
|
12
9
|
require 'sidekiq_utils/web_extensions/memory_monitor'
|
13
10
|
require 'sidekiq_utils/web_extensions/throughput_monitor'
|
14
11
|
|
@@ -17,5 +14,4 @@ require 'sidekiq_utils/additional_serialization'
|
|
17
14
|
require 'sidekiq_utils/deprioritize'
|
18
15
|
require 'sidekiq_utils/enqueued_jobs_helper'
|
19
16
|
require 'sidekiq_utils/find_optional'
|
20
|
-
require 'sidekiq_utils/job_counter'
|
21
17
|
require 'sidekiq_utils/latency_alert'
|
Binary file
|
data/sidekiq_utils.gemspec
CHANGED
@@ -2,16 +2,16 @@
|
|
2
2
|
# DO NOT EDIT THIS FILE DIRECTLY
|
3
3
|
# Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
|
4
4
|
# -*- encoding: utf-8 -*-
|
5
|
-
# stub: sidekiq_utils 1.0.
|
5
|
+
# stub: sidekiq_utils 1.0.5 ruby lib
|
6
6
|
|
7
7
|
Gem::Specification.new do |s|
|
8
8
|
s.name = "sidekiq_utils".freeze
|
9
|
-
s.version = "1.0.
|
9
|
+
s.version = "1.0.5"
|
10
10
|
|
11
11
|
s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
|
12
12
|
s.require_paths = ["lib".freeze]
|
13
13
|
s.authors = ["Magnus von Koeller".freeze]
|
14
|
-
s.date = "
|
14
|
+
s.date = "2019-01-23"
|
15
15
|
s.description = "Tools that make working with a major Sidekiq installation more fun.".freeze
|
16
16
|
s.email = "magnus@angel.co".freeze
|
17
17
|
s.extra_rdoc_files = [
|
@@ -19,6 +19,8 @@ Gem::Specification.new do |s|
|
|
19
19
|
"README.md"
|
20
20
|
]
|
21
21
|
s.files = [
|
22
|
+
".circleci/config.yml",
|
23
|
+
".rspec",
|
22
24
|
"Gemfile",
|
23
25
|
"Gemfile.lock",
|
24
26
|
"LICENSE",
|
@@ -30,28 +32,26 @@ Gem::Specification.new do |s|
|
|
30
32
|
"lib/sidekiq_utils/deprioritize.rb",
|
31
33
|
"lib/sidekiq_utils/enqueued_jobs_helper.rb",
|
32
34
|
"lib/sidekiq_utils/find_optional.rb",
|
33
|
-
"lib/sidekiq_utils/job_counter.rb",
|
34
35
|
"lib/sidekiq_utils/latency_alert.rb",
|
36
|
+
"lib/sidekiq_utils/middleware/.DS_Store",
|
35
37
|
"lib/sidekiq_utils/middleware/client/additional_serialization.rb",
|
36
38
|
"lib/sidekiq_utils/middleware/client/deprioritize.rb",
|
37
|
-
"lib/sidekiq_utils/middleware/client/job_counter.rb",
|
38
39
|
"lib/sidekiq_utils/middleware/server/additional_serialization.rb",
|
39
40
|
"lib/sidekiq_utils/middleware/server/find_optional.rb",
|
40
|
-
"lib/sidekiq_utils/middleware/server/job_counter.rb",
|
41
41
|
"lib/sidekiq_utils/middleware/server/memory_monitor.rb",
|
42
42
|
"lib/sidekiq_utils/middleware/server/throughput_monitor.rb",
|
43
43
|
"lib/sidekiq_utils/redis_monitor_storage.rb",
|
44
|
-
"lib/sidekiq_utils/web_extensions/job_counter.rb",
|
45
44
|
"lib/sidekiq_utils/web_extensions/memory_monitor.rb",
|
46
45
|
"lib/sidekiq_utils/web_extensions/throughput_monitor.rb",
|
47
46
|
"lib/sidekiq_utils/web_extensions/views/job_counts.erb",
|
48
47
|
"lib/sidekiq_utils/web_extensions/views/memory.erb",
|
49
48
|
"lib/sidekiq_utils/web_extensions/views/throughput.erb",
|
50
|
-
"sidekiq_utils.gemspec"
|
49
|
+
"sidekiq_utils.gemspec",
|
50
|
+
"spec/spec_helper.rb"
|
51
51
|
]
|
52
52
|
s.homepage = "http://github.com/venturehacks/sidekiq_angels".freeze
|
53
53
|
s.licenses = ["MIT".freeze]
|
54
|
-
s.rubygems_version = "2.6.
|
54
|
+
s.rubygems_version = "2.6.14.3".freeze
|
55
55
|
s.summary = "Tools that make working with a major Sidekiq installation more fun.".freeze
|
56
56
|
|
57
57
|
if s.respond_to? :specification_version then
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,102 @@
|
|
1
|
+
ENV['RAILS_ENV'] ||= 'test'
|
2
|
+
|
3
|
+
# This file was generated by the `rspec --init` command. Conventionally, all
|
4
|
+
# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
|
5
|
+
# The generated `.rspec` file contains `--require spec_helper` which will cause
|
6
|
+
# this file to always be loaded, without a need to explicitly require it in any
|
7
|
+
# files.
|
8
|
+
#
|
9
|
+
# Given that it is always loaded, you are encouraged to keep this file as
|
10
|
+
# light-weight as possible. Requiring heavyweight dependencies from this file
|
11
|
+
# will add to the boot time of your test suite on EVERY test run, even for an
|
12
|
+
# individual file that may not need all of that loaded. Instead, consider making
|
13
|
+
# a separate helper file that requires the additional dependencies and performs
|
14
|
+
# the additional setup, and require it from the spec files that actually need
|
15
|
+
# it.
|
16
|
+
#
|
17
|
+
# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
|
18
|
+
RSpec.configure do |config|
|
19
|
+
# rspec-expectations config goes here. You can use an alternate
|
20
|
+
# assertion/expectation library such as wrong or the stdlib/minitest
|
21
|
+
# assertions if you prefer.
|
22
|
+
config.expect_with :rspec do |expectations|
|
23
|
+
# This option will default to `true` in RSpec 4. It makes the `description`
|
24
|
+
# and `failure_message` of custom matchers include text for helper methods
|
25
|
+
# defined using `chain`, e.g.:
|
26
|
+
# be_bigger_than(2).and_smaller_than(4).description
|
27
|
+
# # => "be bigger than 2 and smaller than 4"
|
28
|
+
# ...rather than:
|
29
|
+
# # => "be bigger than 2"
|
30
|
+
expectations.include_chain_clauses_in_custom_matcher_descriptions = true
|
31
|
+
end
|
32
|
+
|
33
|
+
# rspec-mocks config goes here. You can use an alternate test double
|
34
|
+
# library (such as bogus or mocha) by changing the `mock_with` option here.
|
35
|
+
config.mock_with :rspec do |mocks|
|
36
|
+
# Prevents you from mocking or stubbing a method that does not exist on
|
37
|
+
# a real object. This is generally recommended, and will default to
|
38
|
+
# `true` in RSpec 4.
|
39
|
+
mocks.verify_partial_doubles = true
|
40
|
+
end
|
41
|
+
|
42
|
+
# This option will default to `:apply_to_host_groups` in RSpec 4 (and will
|
43
|
+
# have no way to turn it off -- the option exists only for backwards
|
44
|
+
# compatibility in RSpec 3). It causes shared context metadata to be
|
45
|
+
# inherited by the metadata hash of host groups and examples, rather than
|
46
|
+
# triggering implicit auto-inclusion in groups with matching metadata.
|
47
|
+
config.shared_context_metadata_behavior = :apply_to_host_groups
|
48
|
+
|
49
|
+
# The settings below are suggested to provide a good initial experience
|
50
|
+
# with RSpec, but feel free to customize to your heart's content.
|
51
|
+
=begin
|
52
|
+
# This allows you to limit a spec run to individual examples or groups
|
53
|
+
# you care about by tagging them with `:focus` metadata. When nothing
|
54
|
+
# is tagged with `:focus`, all examples get run. RSpec also provides
|
55
|
+
# aliases for `it`, `describe`, and `context` that include `:focus`
|
56
|
+
# metadata: `fit`, `fdescribe` and `fcontext`, respectively.
|
57
|
+
config.filter_run_when_matching :focus
|
58
|
+
|
59
|
+
# Allows RSpec to persist some state between runs in order to support
|
60
|
+
# the `--only-failures` and `--next-failure` CLI options. We recommend
|
61
|
+
# you configure your source control system to ignore this file.
|
62
|
+
config.example_status_persistence_file_path = "spec/examples.txt"
|
63
|
+
|
64
|
+
# Limits the available syntax to the non-monkey patched syntax that is
|
65
|
+
# recommended. For more details, see:
|
66
|
+
# - http://rspec.info/blog/2012/06/rspecs-new-expectation-syntax/
|
67
|
+
# - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
|
68
|
+
# - http://rspec.info/blog/2014/05/notable-changes-in-rspec-3/#zero-monkey-patching-mode
|
69
|
+
config.disable_monkey_patching!
|
70
|
+
|
71
|
+
# This setting enables warnings. It's recommended, but in some cases may
|
72
|
+
# be too noisy due to issues in dependencies.
|
73
|
+
config.warnings = true
|
74
|
+
|
75
|
+
# Many RSpec users commonly either run the entire suite or an individual
|
76
|
+
# file, and it's useful to allow more verbose output when running an
|
77
|
+
# individual spec file.
|
78
|
+
if config.files_to_run.one?
|
79
|
+
# Use the documentation formatter for detailed output,
|
80
|
+
# unless a formatter has already been configured
|
81
|
+
# (e.g. via a command-line flag).
|
82
|
+
config.default_formatter = "doc"
|
83
|
+
end
|
84
|
+
|
85
|
+
# Print the 10 slowest examples and example groups at the
|
86
|
+
# end of the spec run, to help surface which specs are running
|
87
|
+
# particularly slow.
|
88
|
+
config.profile_examples = 10
|
89
|
+
|
90
|
+
# Run specs in random order to surface order dependencies. If you find an
|
91
|
+
# order dependency and want to debug it, you can fix the order by providing
|
92
|
+
# the seed, which is printed after each run.
|
93
|
+
# --seed 1234
|
94
|
+
config.order = :random
|
95
|
+
|
96
|
+
# Seed global randomization in this process using the `--seed` CLI option.
|
97
|
+
# Setting this allows you to use `--seed` to deterministically reproduce
|
98
|
+
# test failures related to randomization by passing the same `--seed` value
|
99
|
+
# as the one that triggered the failure.
|
100
|
+
Kernel.srand config.seed
|
101
|
+
=end
|
102
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sidekiq_utils
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Magnus von Koeller
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2019-01-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: sidekiq
|
@@ -102,6 +102,8 @@ extra_rdoc_files:
|
|
102
102
|
- LICENSE
|
103
103
|
- README.md
|
104
104
|
files:
|
105
|
+
- ".circleci/config.yml"
|
106
|
+
- ".rspec"
|
105
107
|
- Gemfile
|
106
108
|
- Gemfile.lock
|
107
109
|
- LICENSE
|
@@ -113,24 +115,22 @@ files:
|
|
113
115
|
- lib/sidekiq_utils/deprioritize.rb
|
114
116
|
- lib/sidekiq_utils/enqueued_jobs_helper.rb
|
115
117
|
- lib/sidekiq_utils/find_optional.rb
|
116
|
-
- lib/sidekiq_utils/job_counter.rb
|
117
118
|
- lib/sidekiq_utils/latency_alert.rb
|
119
|
+
- lib/sidekiq_utils/middleware/.DS_Store
|
118
120
|
- lib/sidekiq_utils/middleware/client/additional_serialization.rb
|
119
121
|
- lib/sidekiq_utils/middleware/client/deprioritize.rb
|
120
|
-
- lib/sidekiq_utils/middleware/client/job_counter.rb
|
121
122
|
- lib/sidekiq_utils/middleware/server/additional_serialization.rb
|
122
123
|
- lib/sidekiq_utils/middleware/server/find_optional.rb
|
123
|
-
- lib/sidekiq_utils/middleware/server/job_counter.rb
|
124
124
|
- lib/sidekiq_utils/middleware/server/memory_monitor.rb
|
125
125
|
- lib/sidekiq_utils/middleware/server/throughput_monitor.rb
|
126
126
|
- lib/sidekiq_utils/redis_monitor_storage.rb
|
127
|
-
- lib/sidekiq_utils/web_extensions/job_counter.rb
|
128
127
|
- lib/sidekiq_utils/web_extensions/memory_monitor.rb
|
129
128
|
- lib/sidekiq_utils/web_extensions/throughput_monitor.rb
|
130
129
|
- lib/sidekiq_utils/web_extensions/views/job_counts.erb
|
131
130
|
- lib/sidekiq_utils/web_extensions/views/memory.erb
|
132
131
|
- lib/sidekiq_utils/web_extensions/views/throughput.erb
|
133
132
|
- sidekiq_utils.gemspec
|
133
|
+
- spec/spec_helper.rb
|
134
134
|
homepage: http://github.com/venturehacks/sidekiq_angels
|
135
135
|
licenses:
|
136
136
|
- MIT
|
@@ -151,7 +151,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
151
151
|
version: '0'
|
152
152
|
requirements: []
|
153
153
|
rubyforge_project:
|
154
|
-
rubygems_version: 2.6.
|
154
|
+
rubygems_version: 2.6.14.3
|
155
155
|
signing_key:
|
156
156
|
specification_version: 4
|
157
157
|
summary: Tools that make working with a major Sidekiq installation more fun.
|
@@ -1,161 +0,0 @@
|
|
1
|
-
module SidekiqUtils
|
2
|
-
module JobCounter
|
3
|
-
REDIS_KEY = 'sidekiq_utils_job_counter'
|
4
|
-
LOCK = Mutex.new
|
5
|
-
SYNC_COUNTS_EVERY = 1.second
|
6
|
-
|
7
|
-
class << self
|
8
|
-
def increment(job)
|
9
|
-
change_count(job, 1)
|
10
|
-
end
|
11
|
-
|
12
|
-
def decrement(job)
|
13
|
-
change_count(job, -1)
|
14
|
-
end
|
15
|
-
|
16
|
-
def counts
|
17
|
-
Sidekiq.redis do |redis|
|
18
|
-
redis.hgetall(REDIS_KEY).each_with_object({}) do |(key, count), ret_hash|
|
19
|
-
key_hash = JSON.parse(key, symbolize_names: true)
|
20
|
-
|
21
|
-
count = count.to_i
|
22
|
-
if count != 0
|
23
|
-
ret_hash[key_hash[:queue]] ||= {}
|
24
|
-
ret_hash[key_hash[:queue]][key_hash[:job]] = count
|
25
|
-
end
|
26
|
-
end
|
27
|
-
end
|
28
|
-
end
|
29
|
-
|
30
|
-
def reset!
|
31
|
-
Sidekiq.redis do |redis|
|
32
|
-
redis.del(REDIS_KEY)
|
33
|
-
end
|
34
|
-
end
|
35
|
-
|
36
|
-
def hook_sidekiq!
|
37
|
-
Sidekiq::SortedEntry.prepend(SortedEntry::JobCounterExtension)
|
38
|
-
Sidekiq::ScheduledSet.prepend(ScheduledSet::JobCounterExtension)
|
39
|
-
Sidekiq::Job.prepend(Job::JobCounterExtension)
|
40
|
-
Sidekiq::Queue.prepend(Queue::JobCounterExtension)
|
41
|
-
Sidekiq::JobSet.prepend(JobSet::JobCounterExtension)
|
42
|
-
end
|
43
|
-
|
44
|
-
private
|
45
|
-
def change_count(job, change_by)
|
46
|
-
unless [-1, 1].include?(change_by)
|
47
|
-
fail("Unsupported change_by value: #{change_by}")
|
48
|
-
end
|
49
|
-
|
50
|
-
job_key = SidekiqUtils::RedisMonitorStorage.job_prefix(
|
51
|
-
job, unwrap_arguments: true)
|
52
|
-
hash_key = {
|
53
|
-
queue: job['queue'],
|
54
|
-
job: job_key,
|
55
|
-
}.to_json
|
56
|
-
|
57
|
-
LOCK.synchronize do
|
58
|
-
@counts_to_flush ||= {}
|
59
|
-
@counts_to_flush[hash_key] ||= 0
|
60
|
-
@counts_to_flush[hash_key] += change_by
|
61
|
-
|
62
|
-
if !Rails.env.test?
|
63
|
-
@sync_thread ||= Thread.new do
|
64
|
-
sleep SYNC_COUNTS_EVERY
|
65
|
-
sync_to_redis
|
66
|
-
end
|
67
|
-
end
|
68
|
-
end
|
69
|
-
|
70
|
-
sync_to_redis if Rails.env.test?
|
71
|
-
end
|
72
|
-
|
73
|
-
def sync_to_redis
|
74
|
-
local_counts_to_flush = nil
|
75
|
-
LOCK.synchronize do
|
76
|
-
local_counts_to_flush = @counts_to_flush
|
77
|
-
@counts_to_flush = {}
|
78
|
-
@sync_thread = nil
|
79
|
-
end
|
80
|
-
(local_counts_to_flush || {}).each do |hash_key, change_by|
|
81
|
-
Sidekiq.redis do |redis|
|
82
|
-
count = redis.hincrby(REDIS_KEY, hash_key, change_by)
|
83
|
-
if count < 0 && change_by < 0
|
84
|
-
# this shouldn't happen, but it could when we first deploy this.
|
85
|
-
# just makes sure we don't end up with negative values here
|
86
|
-
# which don't make sense
|
87
|
-
#
|
88
|
-
# we only ever increment by the same amount we just did because
|
89
|
-
# we can't be responsible for more of a discrepancy at this
|
90
|
-
# point and we don't want multiple threads to overcorrect for
|
91
|
-
# each other
|
92
|
-
count = redis.hincrby(REDIS_KEY, hash_key, -1*change_by)
|
93
|
-
end
|
94
|
-
end
|
95
|
-
end
|
96
|
-
end
|
97
|
-
at_exit { SidekiqUtils::JobCounter.send(:sync_to_redis) }
|
98
|
-
end
|
99
|
-
end
|
100
|
-
|
101
|
-
class SortedEntry
|
102
|
-
module JobCounterExtension
|
103
|
-
def delete
|
104
|
-
if super
|
105
|
-
JobCounter.decrement(item)
|
106
|
-
end
|
107
|
-
end
|
108
|
-
|
109
|
-
private
|
110
|
-
|
111
|
-
def remove_job
|
112
|
-
super do |message|
|
113
|
-
JobCounter.decrement(Sidekiq.load_json(message))
|
114
|
-
yield message
|
115
|
-
end
|
116
|
-
end
|
117
|
-
end
|
118
|
-
end
|
119
|
-
|
120
|
-
class ScheduledSet
|
121
|
-
module JobCounterExtension
|
122
|
-
def delete
|
123
|
-
if super
|
124
|
-
JobCounter.decrement(item)
|
125
|
-
end
|
126
|
-
end
|
127
|
-
end
|
128
|
-
end
|
129
|
-
|
130
|
-
class Job
|
131
|
-
module JobCounterExtension
|
132
|
-
def delete
|
133
|
-
super
|
134
|
-
JobCounter.decrement(item)
|
135
|
-
end
|
136
|
-
end
|
137
|
-
end
|
138
|
-
|
139
|
-
class Queue
|
140
|
-
module JobCounterExtension
|
141
|
-
def clear
|
142
|
-
super
|
143
|
-
end
|
144
|
-
end
|
145
|
-
end
|
146
|
-
|
147
|
-
class JobSet
|
148
|
-
module JobCounterExtension
|
149
|
-
def clear
|
150
|
-
each(&:delete)
|
151
|
-
super
|
152
|
-
end
|
153
|
-
|
154
|
-
def delete_by_value(name, value)
|
155
|
-
if super
|
156
|
-
JobCounter.decrement(Sidekiq.load_json(value))
|
157
|
-
end
|
158
|
-
end
|
159
|
-
end
|
160
|
-
end
|
161
|
-
end
|
@@ -1,17 +0,0 @@
|
|
1
|
-
module SidekiqUtils
|
2
|
-
module Middleware
|
3
|
-
module Client
|
4
|
-
class JobCounter
|
5
|
-
def call(worker_class, job, queue, redis_pool)
|
6
|
-
unless job['at']
|
7
|
-
# don't count when jobs get put on the scheduled set, because
|
8
|
-
# otherwise we'll double-count them when they get popped and moved
|
9
|
-
# to a work queue.
|
10
|
-
SidekiqUtils::JobCounter.increment(job)
|
11
|
-
end
|
12
|
-
yield
|
13
|
-
end
|
14
|
-
end
|
15
|
-
end
|
16
|
-
end
|
17
|
-
end
|
@@ -1,15 +0,0 @@
|
|
1
|
-
module SidekiqUtils
|
2
|
-
module Middleware
|
3
|
-
module Server
|
4
|
-
class JobCounter
|
5
|
-
def call(worker, job, queue)
|
6
|
-
# we decrement here whether the job succeeds or not, because
|
7
|
-
# re-enqueuing from the retry queue triggers the client middleware
|
8
|
-
# and thus another increment even in the case of an error
|
9
|
-
SidekiqUtils::JobCounter.decrement(job)
|
10
|
-
yield
|
11
|
-
end
|
12
|
-
end
|
13
|
-
end
|
14
|
-
end
|
15
|
-
end
|
@@ -1,35 +0,0 @@
|
|
1
|
-
module SidekiqUtils
|
2
|
-
module WebExtensions
|
3
|
-
module JobCounter
|
4
|
-
def self.registered(app)
|
5
|
-
view_path = File.join(File.expand_path("..", __FILE__), "views")
|
6
|
-
|
7
|
-
require 'active_support/number_helper'
|
8
|
-
app.get("/job_counts") do
|
9
|
-
@throughput = SidekiqUtils::RedisMonitorStorage.
|
10
|
-
retrieve('sidekiq_elapsed', 'elapsed')
|
11
|
-
@counts = []
|
12
|
-
SidekiqUtils::JobCounter.counts.each do |queue, job_counts|
|
13
|
-
job_counts.each do |job, count|
|
14
|
-
values = { queue: queue, job: job, count: count }
|
15
|
-
if (values[:avg_runtime] = @throughput[job].try!(:[], 'average'))
|
16
|
-
execution_time = values[:count] * values[:avg_runtime].to_f / 1_000
|
17
|
-
values[:runtime_day] = days =
|
18
|
-
(execution_time / 1.day).floor
|
19
|
-
values[:runtime_hour] = hours =
|
20
|
-
((execution_time - days.days) / 1.hour).floor
|
21
|
-
values[:runtime_min] = (
|
22
|
-
(execution_time - days.days - hours.hours) / 1.minute
|
23
|
-
).round
|
24
|
-
end
|
25
|
-
@counts << values
|
26
|
-
end
|
27
|
-
end
|
28
|
-
@counts.sort_by! {|x| -1 * x[:count] }
|
29
|
-
|
30
|
-
render(:erb, File.read(File.join(view_path, "job_counts.erb")))
|
31
|
-
end
|
32
|
-
end
|
33
|
-
end
|
34
|
-
end
|
35
|
-
end
|