sidekiq-ultimate 0.0.1.alpha.18 → 2.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/rspec.yml +58 -0
- data/.gitignore +2 -0
- data/.rubocop.yml +42 -6
- data/ARCHITECTURE.md +1 -1
- data/Appraisals +13 -0
- data/CHANGES.md +6 -0
- data/Gemfile +5 -7
- data/README.md +80 -14
- data/Rakefile +1 -5
- data/docker-compose.yml +12 -0
- data/gemfiles/redis_4.8.0_sidekiq_6.5.0.gemfile +29 -0
- data/lib/sidekiq/ultimate/configuration.rb +67 -0
- data/lib/sidekiq/ultimate/empty_queues/refresh_timer_task.rb +26 -0
- data/lib/sidekiq/ultimate/empty_queues.rb +144 -0
- data/lib/sidekiq/ultimate/expirable_set.rb +2 -5
- data/lib/sidekiq/ultimate/fetch.rb +29 -23
- data/lib/sidekiq/ultimate/interval_with_jitter.rb +19 -0
- data/lib/sidekiq/ultimate/queue_name.rb +4 -5
- data/lib/sidekiq/ultimate/resurrector/common_constants.rb +13 -0
- data/lib/sidekiq/ultimate/resurrector/count.rb +22 -0
- data/lib/sidekiq/ultimate/resurrector/lock.rb +46 -0
- data/lib/sidekiq/ultimate/resurrector/lua_scripts/resurrect_with_counter.lua +22 -0
- data/lib/sidekiq/ultimate/resurrector/resurrection_script.rb +51 -0
- data/lib/sidekiq/ultimate/resurrector.rb +62 -69
- data/lib/sidekiq/ultimate/unit_of_work.rb +5 -4
- data/lib/sidekiq/ultimate/version.rb +1 -2
- data/lib/sidekiq/ultimate.rb +11 -2
- data/sidekiq-ultimate.gemspec +8 -10
- metadata +51 -24
- data/.travis.yml +0 -28
- /data/lib/sidekiq/ultimate/resurrector/{resurrect.lua → lua_scripts/resurrect.lua} +0 -0
- /data/lib/sidekiq/ultimate/resurrector/{safeclean.lua → lua_scripts/safeclean.lua} +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3fe29c4188e4bcc73a818ac756cc6fa0eaa97588c2373920d0dc9cf36ba8859e
|
4
|
+
data.tar.gz: bb650bdf494eab357b0d042243cb42d9e5c1a0914e70971576766a55c84508d7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9389d82c1071237e11955ea3ad62ada63c084d4a41d412c87ab8b034982dd0104a4ee4701dbf09e563322150234cc06c18150be65ea5d51aeec3a38b03a55964
|
7
|
+
data.tar.gz: 6a0d8801f2959479e77ec201d377dfac040eacf5a366237962622faf4b92e9696d3cfed40ee00f8b03893798da848e5b42bfa48af27e3493020c87bcb20c7821
|
@@ -0,0 +1,58 @@
|
|
1
|
+
# This workflow uses actions that are not certified by GitHub.
|
2
|
+
# They are provided by a third-party and are governed by
|
3
|
+
# separate terms of service, privacy policy, and support
|
4
|
+
# documentation.
|
5
|
+
|
6
|
+
# GitHub recommends pinning actions to a commit SHA.
|
7
|
+
# To get a newer version, you will need to update the SHA.
|
8
|
+
# You can also reference a tag or branch, but the action may change without warning.
|
9
|
+
|
10
|
+
name: Gem Rspec testing
|
11
|
+
|
12
|
+
on:
|
13
|
+
push:
|
14
|
+
branches: [ master ]
|
15
|
+
pull_request:
|
16
|
+
branches: [ master ]
|
17
|
+
|
18
|
+
permissions:
|
19
|
+
contents: read
|
20
|
+
|
21
|
+
jobs:
|
22
|
+
rubocop:
|
23
|
+
name: Rubocop
|
24
|
+
runs-on: ubuntu-latest
|
25
|
+
env:
|
26
|
+
BUNDLE_WITHOUT: development
|
27
|
+
steps:
|
28
|
+
- uses: actions/checkout@v3
|
29
|
+
- uses: ruby/setup-ruby@v1
|
30
|
+
with:
|
31
|
+
ruby-version: 2.7.6
|
32
|
+
bundler-cache: true
|
33
|
+
- run: bundle exec rubocop --config .rubocop.yml
|
34
|
+
|
35
|
+
test:
|
36
|
+
runs-on: ubuntu-latest
|
37
|
+
strategy:
|
38
|
+
fail-fast: false
|
39
|
+
matrix:
|
40
|
+
ruby-version: ['2.7.6']
|
41
|
+
redis: ['4.8.0']
|
42
|
+
sidekiq: ['6.5.0']
|
43
|
+
env:
|
44
|
+
BUNDLE_GEMFILE: gemfiles/redis_${{ matrix.redis }}_sidekiq_${{ matrix.sidekiq }}.gemfile
|
45
|
+
BUNDLE_WITHOUT: development
|
46
|
+
|
47
|
+
steps:
|
48
|
+
- uses: actions/checkout@v3
|
49
|
+
- name: Set up Ruby
|
50
|
+
uses: ruby/setup-ruby@v1
|
51
|
+
with:
|
52
|
+
ruby-version: ${{ matrix.ruby-version }}
|
53
|
+
bundler-cache: true # runs 'bundle install' and caches installed gems automatically
|
54
|
+
- run: docker-compose up -d
|
55
|
+
- name: Run specs and generate coverage report
|
56
|
+
run: bundle exec rake spec
|
57
|
+
- name: Upload coverage to Codecov
|
58
|
+
uses: codecov/codecov-action@v3
|
data/.gitignore
CHANGED
data/.rubocop.yml
CHANGED
@@ -1,6 +1,15 @@
|
|
1
|
+
require:
|
2
|
+
- rubocop-performance
|
3
|
+
- rubocop-rake
|
4
|
+
- rubocop-rspec
|
5
|
+
|
1
6
|
AllCops:
|
2
|
-
TargetRubyVersion: 2.
|
7
|
+
TargetRubyVersion: 2.7
|
3
8
|
DisplayCopNames: true
|
9
|
+
NewCops: enable
|
10
|
+
Exclude:
|
11
|
+
- "gemfiles/**/*"
|
12
|
+
- "vendor/**/*"
|
4
13
|
|
5
14
|
|
6
15
|
## Layout ######################################################################
|
@@ -8,12 +17,17 @@ AllCops:
|
|
8
17
|
Layout/DotPosition:
|
9
18
|
EnforcedStyle: trailing
|
10
19
|
|
11
|
-
Layout/
|
20
|
+
Layout/FirstArrayElementIndentation:
|
12
21
|
EnforcedStyle: consistent
|
13
22
|
|
14
|
-
Layout/
|
23
|
+
Layout/FirstHashElementIndentation:
|
15
24
|
EnforcedStyle: consistent
|
16
25
|
|
26
|
+
Layout/LineLength:
|
27
|
+
Max: 120
|
28
|
+
|
29
|
+
Layout/HashAlignment:
|
30
|
+
EnforcedHashRocketStyle: table
|
17
31
|
|
18
32
|
## Metrics #####################################################################
|
19
33
|
|
@@ -21,11 +35,13 @@ Metrics/BlockLength:
|
|
21
35
|
Exclude:
|
22
36
|
- "spec/**/*"
|
23
37
|
|
38
|
+
Metrics/MethodLength:
|
39
|
+
CountAsOne: ['array', 'hash', 'heredoc', 'method_call']
|
24
40
|
|
25
|
-
|
41
|
+
Metrics/ModuleLength:
|
42
|
+
CountAsOne: ['array', 'hash', 'heredoc', 'method_call']
|
26
43
|
|
27
|
-
Style
|
28
|
-
Enabled: false
|
44
|
+
## Style #######################################################################
|
29
45
|
|
30
46
|
Style/HashSyntax:
|
31
47
|
EnforcedStyle: hash_rockets
|
@@ -44,3 +60,23 @@ Style/StringLiterals:
|
|
44
60
|
|
45
61
|
Style/YodaCondition:
|
46
62
|
Enabled: false
|
63
|
+
|
64
|
+
## RSpec ########################################################################
|
65
|
+
RSpec/ExampleLength:
|
66
|
+
Enabled: false
|
67
|
+
|
68
|
+
RSpec/MultipleExpectations:
|
69
|
+
Enabled: false
|
70
|
+
|
71
|
+
RSpec/MultipleMemoizedHelpers:
|
72
|
+
Enabled: false
|
73
|
+
|
74
|
+
RSpec/NestedGroups:
|
75
|
+
Enabled: false
|
76
|
+
|
77
|
+
Lint/AmbiguousBlockAssociation:
|
78
|
+
Exclude:
|
79
|
+
- "spec/**/*"
|
80
|
+
|
81
|
+
RSpec/IndexedLet:
|
82
|
+
Max: 5
|
data/ARCHITECTURE.md
CHANGED
data/Appraisals
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
REDIS_VERSIONS = %w[4.8.0].freeze
|
4
|
+
SIDEKIQ_VERSIONS = %w[6.5.0].freeze
|
5
|
+
|
6
|
+
version_combinations = REDIS_VERSIONS.product(SIDEKIQ_VERSIONS)
|
7
|
+
|
8
|
+
version_combinations.each do |redis_version, sidekiq_version|
|
9
|
+
appraise "redis_#{redis_version}_sidekiq_#{sidekiq_version}" do
|
10
|
+
gem "redis", "~> #{redis_version}"
|
11
|
+
gem "sidekiq", "~> #{sidekiq_version}"
|
12
|
+
end
|
13
|
+
end
|
data/CHANGES.md
ADDED
data/Gemfile
CHANGED
@@ -3,9 +3,13 @@
|
|
3
3
|
source "https://rubygems.org"
|
4
4
|
ruby RUBY_VERSION
|
5
5
|
|
6
|
+
gem "appraisal"
|
6
7
|
gem "rake"
|
7
8
|
gem "rspec"
|
8
|
-
gem "rubocop", "~>
|
9
|
+
gem "rubocop", "~> 1.50.2", :require => false
|
10
|
+
gem "rubocop-performance", "~> 1.17.1", :require => false
|
11
|
+
gem "rubocop-rake", "~> 0.6.0", :require => false
|
12
|
+
gem "rubocop-rspec", "~> 2.20.0", :require => false
|
9
13
|
|
10
14
|
group :development do
|
11
15
|
gem "guard", :require => false
|
@@ -19,10 +23,4 @@ group :test do
|
|
19
23
|
gem "simplecov", :require => false
|
20
24
|
end
|
21
25
|
|
22
|
-
group :doc do
|
23
|
-
gem "redcarpet"
|
24
|
-
gem "yard"
|
25
|
-
end
|
26
|
-
|
27
|
-
# Specify your gem's dependencies in redis-prescription.gemspec
|
28
26
|
gemspec
|
data/README.md
CHANGED
@@ -18,7 +18,7 @@ extension one will need. :D
|
|
18
18
|
Add this line to your application's Gemfile:
|
19
19
|
|
20
20
|
```ruby
|
21
|
-
gem "sidekiq-ultimate"
|
21
|
+
gem "sidekiq-ultimate"
|
22
22
|
```
|
23
23
|
|
24
24
|
And then execute:
|
@@ -40,6 +40,80 @@ require "sidekiq/ultimate"
|
|
40
40
|
Sidekiq::Ultimate.setup!
|
41
41
|
```
|
42
42
|
|
43
|
+
## Configuration
|
44
|
+
|
45
|
+
### Resurrection event handler
|
46
|
+
|
47
|
+
An event handler can be called when a job is resurrected.
|
48
|
+
|
49
|
+
```ruby
|
50
|
+
Sidekiq::Ultimate.setup! do |config|
|
51
|
+
config.on_resurrection = ->(queue_name, jobs_count) do
|
52
|
+
puts "Resurrected #{jobs_count} jobs from #{queue_name}"
|
53
|
+
end
|
54
|
+
end
|
55
|
+
```
|
56
|
+
|
57
|
+
### Resurrection counter
|
58
|
+
|
59
|
+
A resurrection counter can be enabled to count how many times a job was resurrected. If `enable_resurrection_counter` setting is enabled, on each resurrection event, a counter is increased. Counter value is stored in redis and has expiration time 24 hours.
|
60
|
+
|
61
|
+
For example this can be used in the `ServerMiddleware` later on to early return resurrected jobs based on the counter value.
|
62
|
+
|
63
|
+
`enable_resurrection_counter` can be either a `Proc` or a constant.
|
64
|
+
|
65
|
+
Having a `Proc` is useful if you want to enable or disable resurrection counter in run time. It will be called on each
|
66
|
+
resurrection event to decide whether to increase the counter or not.
|
67
|
+
|
68
|
+
```ruby
|
69
|
+
Sidekiq::Ultimate.setup! do |config|
|
70
|
+
config.enable_resurrection_counter = -> do
|
71
|
+
DynamicSettings.get("enable_resurrection_counter")
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
Sidekiq::Ultimate.setup! do |config|
|
76
|
+
config.enable_resurrection_counter = true
|
77
|
+
end
|
78
|
+
```
|
79
|
+
|
80
|
+
#### Read the value
|
81
|
+
|
82
|
+
Resurrection counter value can be read using `Sidekiq::Ultimate::Resurrector::Count.read` method.
|
83
|
+
|
84
|
+
```ruby
|
85
|
+
Sidekiq::Ultimate::Resurrector::Count.read(:job_id => "2647c4fe13acc692326bd4c2")
|
86
|
+
=> 1
|
87
|
+
```
|
88
|
+
|
89
|
+
### Empty Queues Cache Refresh Interval
|
90
|
+
|
91
|
+
```ruby
|
92
|
+
Sidekiq::Ultimate.setup! do |config|
|
93
|
+
config.empty_queues_cache_refresh_interval_sec = 42
|
94
|
+
end
|
95
|
+
```
|
96
|
+
|
97
|
+
Specifies how often the cache of empty queues should be refreshed.
|
98
|
+
In a nutshell, this sets the maximum possible delay between when a job was pushed to previously empty queue and earliest the moment when that new job could be picked up.
|
99
|
+
|
100
|
+
**Note:** every sidekiq process maintains its own local cache of empty queues.
|
101
|
+
Setting this interval to a low value will increase the number of Redis calls needed to check for empty queues, increasing the total load on Redis.
|
102
|
+
|
103
|
+
This setting helps manage the tradeoff between performance penalties and latency needed for reliable fetch.
|
104
|
+
Under the hood, Sidekiq's default fetch occurs with [a single Redis `BRPOP` call](https://redis.io/commands/brpop/) which is passes list of all queues to pluck work from.
|
105
|
+
In contrast, [reliable fetch uses `LPOPRPUSH`](https://redis.io/commands/rpoplpush/) (or the equivalent `LMOVE` in later Redis versions) to place in progress work into a WIP queue.
|
106
|
+
However, `LPOPRPUSH` can only check one source queue to pop from at once, and [no multi-key alternative is available](https://github.com/redis/redis/issues/1785), so multiple Redis calls are needed to pluck work if an empty queue is checked.
|
107
|
+
In order to avoid performance penalties for repeated calls to empty queues, Sidekiq Ultimate therefore maintains a list of recently know empty queues which it will avoid polling for work.
|
108
|
+
|
109
|
+
Therefore:
|
110
|
+
- If your Sidekiq architecture has *a low number of total queues*, the worst case penalty for polling empty queues will be bounded, and it is reasonable to **set a shorter refresh period**.
|
111
|
+
- If your Sidekiq architecture has a *high number of total queues*, the worst case penalty for polling empty queues is large, and it is recommended to **set a longer refresh period**.
|
112
|
+
- When adjusting this setting:
|
113
|
+
- Check that work is consumed appropriately quickly from high priority queues after they bottom out (after increasing the refresh interval)
|
114
|
+
- Check that backlog work does not accumulate in low priority queues (after decreasing the refresh interval)
|
115
|
+
|
116
|
+
|
43
117
|
---
|
44
118
|
|
45
119
|
**NOTICE**
|
@@ -54,20 +128,13 @@ Thus look up it's README for throttling configuration details.
|
|
54
128
|
|
55
129
|
## Supported Ruby Versions
|
56
130
|
|
57
|
-
This library aims to support and is
|
58
|
-
Ruby and Redis client versions:
|
131
|
+
This library aims to support and is tested against the following Ruby and Redis client versions:
|
59
132
|
|
60
133
|
* Ruby
|
61
|
-
* 2.
|
62
|
-
* 2.4.x
|
63
|
-
* 2.5.x
|
134
|
+
* 2.7.x
|
64
135
|
|
65
136
|
* [redis-rb](https://github.com/redis/redis-rb)
|
66
|
-
* 4.
|
67
|
-
|
68
|
-
* [redis-namespace](https://github.com/resque/redis-namespace)
|
69
|
-
* 1.6
|
70
|
-
|
137
|
+
* 4.8
|
71
138
|
|
72
139
|
If something doesn't work on one of these versions, it's a bug.
|
73
140
|
|
@@ -106,11 +173,10 @@ push git commits and tags, and push the `.gem` file to [rubygems.org][].
|
|
106
173
|
|
107
174
|
## Copyright
|
108
175
|
|
109
|
-
Copyright (c) 2018 SensorTower Inc.<br>
|
176
|
+
Copyright (c) 2018-23 SensorTower Inc.<br>
|
110
177
|
See [LICENSE.md][] for further details.
|
111
178
|
|
112
179
|
|
113
|
-
[travis.ci]: http://travis-ci.org/sensortower/sidekiq-ultimate
|
114
180
|
[rubygems.org]: https://rubygems.org
|
115
181
|
[LICENSE.md]: https://github.com/sensortower/sidekiq-ultimate/blob/master/LICENSE.txt
|
116
|
-
[sidekiq-throttled]:
|
182
|
+
[sidekiq-throttled]: https://github.com/ixti/sidekiq-throttled
|
data/Rakefile
CHANGED
data/docker-compose.yml
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
# This file was generated by Appraisal
|
2
|
+
|
3
|
+
source "https://rubygems.org"
|
4
|
+
|
5
|
+
ruby "2.7.6"
|
6
|
+
|
7
|
+
gem "appraisal"
|
8
|
+
gem "rake"
|
9
|
+
gem "rspec"
|
10
|
+
gem "rubocop", "~> 1.50.2", require: false
|
11
|
+
gem "rubocop-performance", "~> 1.17.1", require: false
|
12
|
+
gem "rubocop-rake", "~> 0.6.0", require: false
|
13
|
+
gem "rubocop-rspec", "~> 2.20.0", require: false
|
14
|
+
gem "redis", "~> 4.8.0"
|
15
|
+
gem "sidekiq", "~> 6.5.0"
|
16
|
+
|
17
|
+
group :development do
|
18
|
+
gem "guard", require: false
|
19
|
+
gem "guard-rspec", require: false
|
20
|
+
gem "guard-rubocop", require: false
|
21
|
+
gem "pry", require: false
|
22
|
+
end
|
23
|
+
|
24
|
+
group :test do
|
25
|
+
gem "codecov", require: false
|
26
|
+
gem "simplecov", require: false
|
27
|
+
end
|
28
|
+
|
29
|
+
gemspec path: "../"
|
@@ -0,0 +1,67 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "singleton"
|
4
|
+
|
5
|
+
module Sidekiq
|
6
|
+
module Ultimate
|
7
|
+
# Configuration options.
|
8
|
+
class Configuration
|
9
|
+
include Singleton
|
10
|
+
|
11
|
+
# @return [Proc] callback to be called when job is resurrected
|
12
|
+
# @yieldparam queue_name [String] name of the queue
|
13
|
+
# @yieldparam jobs_count [Integer] number of jobs resurrected
|
14
|
+
# @yieldreturn [void]
|
15
|
+
# @example
|
16
|
+
# Sidekiq::Ultimate::Configuration.instance.on_resurrection = ->(queue_name, jobs_count) do
|
17
|
+
# puts "Resurrected #{jobs_count} jobs from #{queue_name}"
|
18
|
+
# end
|
19
|
+
attr_accessor :on_resurrection
|
20
|
+
|
21
|
+
# If `enable_resurrection_counter` setting is enabled, on each resurrection event, a counter is increased.
|
22
|
+
# This is useful for telemetry purposes in order to understand how often jobs are resurrected
|
23
|
+
# Counter value is stored in redis by jid and has expiration time 24 hours.
|
24
|
+
# @return [Boolean]
|
25
|
+
attr_accessor :enable_resurrection_counter
|
26
|
+
|
27
|
+
# It specifies how often the cache of empty queues should be refreshed.
|
28
|
+
# In a nutshell, it specifies the maximum possible delay between a job was pushed to previously empty queue and
|
29
|
+
# the moment when that new job is picked up.
|
30
|
+
# Note that every sidekiq process needs to maintain its own local cache of empty queues. Setting this interval
|
31
|
+
# to a low values will increase the number of redis calls and will increase the load on redis.
|
32
|
+
# @return [Numeric] interval in seconds to refresh the cache of empty queues
|
33
|
+
attr_reader :empty_queues_cache_refresh_interval_sec
|
34
|
+
|
35
|
+
DEFAULT_EMPTY_QUEUES_CACHE_REFRESH_INTERVAL_SEC = 30
|
36
|
+
|
37
|
+
# If fetching attempt from a queue was throttled, it puts the queue to the exhausted list for this amount of time
|
38
|
+
# to avoid throttling for the same queue
|
39
|
+
# @return [Float] timeout in seconds
|
40
|
+
attr_writer :throttled_fetch_timeout_sec
|
41
|
+
|
42
|
+
DEFAULT_THROTTLED_FETCH_TIMEOUT_SEC = 15
|
43
|
+
|
44
|
+
def initialize
|
45
|
+
@empty_queues_cache_refresh_interval_sec = DEFAULT_EMPTY_QUEUES_CACHE_REFRESH_INTERVAL_SEC
|
46
|
+
@throttled_fetch_timeout_sec = DEFAULT_THROTTLED_FETCH_TIMEOUT_SEC
|
47
|
+
super
|
48
|
+
end
|
49
|
+
|
50
|
+
def empty_queues_cache_refresh_interval_sec=(value)
|
51
|
+
unless value.is_a?(Numeric)
|
52
|
+
raise ArgumentError, "Invalid 'empty_queues_cache_refresh_interval_sec' value: #{value}. Must be Numeric"
|
53
|
+
end
|
54
|
+
|
55
|
+
@empty_queues_cache_refresh_interval_sec = value
|
56
|
+
end
|
57
|
+
|
58
|
+
def throttled_fetch_timeout_sec
|
59
|
+
if @throttled_fetch_timeout_sec.respond_to?(:call)
|
60
|
+
@throttled_fetch_timeout_sec.call.to_f
|
61
|
+
else
|
62
|
+
@throttled_fetch_timeout_sec.to_f
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "concurrent/timer_task"
|
4
|
+
require "sidekiq/ultimate/interval_with_jitter"
|
5
|
+
|
6
|
+
module Sidekiq
|
7
|
+
module Ultimate
|
8
|
+
class EmptyQueues
|
9
|
+
# Timer task that periodically refreshes empty queues. Also adds jitter to the execution interval.
|
10
|
+
class RefreshTimerTask
|
11
|
+
TASK_CLASS = Class.new(Concurrent::TimerTask)
|
12
|
+
|
13
|
+
class << self
|
14
|
+
def setup!(empty_queues_class)
|
15
|
+
interval = Sidekiq::Ultimate::Configuration.instance.empty_queues_cache_refresh_interval_sec
|
16
|
+
task = TASK_CLASS.new({
|
17
|
+
:run_now => true,
|
18
|
+
:execution_interval => Sidekiq::Ultimate::IntervalWithJitter.call(interval)
|
19
|
+
}) { empty_queues_class.instance.refresh! }
|
20
|
+
task.execute
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,144 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "redlock"
|
4
|
+
require "singleton"
|
5
|
+
|
6
|
+
require "sidekiq/ultimate/configuration"
|
7
|
+
require "sidekiq/ultimate/queue_name"
|
8
|
+
require "sidekiq/ultimate/empty_queues/refresh_timer_task"
|
9
|
+
|
10
|
+
module Sidekiq
|
11
|
+
module Ultimate
|
12
|
+
# Maintains a cache of empty queues. It has a global cache and a local cache.
|
13
|
+
# The global cache is stored in redis and updated periodically. The local cache is updated either by using the fresh
|
14
|
+
# cache fetched after global cache update or by using existing global cache.
|
15
|
+
# Only one process can update the global cache at a time.
|
16
|
+
class EmptyQueues
|
17
|
+
include Singleton
|
18
|
+
|
19
|
+
LOCK_KEY = "ultimate:empty_queues_updater:lock"
|
20
|
+
LAST_RUN_KEY = "ultimate:empty_queues_updater:last_run"
|
21
|
+
KEY = "ultimate:empty_queues"
|
22
|
+
|
23
|
+
attr_reader :queues, :local_lock
|
24
|
+
|
25
|
+
def initialize
|
26
|
+
@local_lock = Mutex.new
|
27
|
+
@queues = []
|
28
|
+
|
29
|
+
super
|
30
|
+
end
|
31
|
+
|
32
|
+
# Sets up automatic empty queues cache updater.
|
33
|
+
# It will call #refresh! every
|
34
|
+
# `Sidekiq::Ultimate::Configuration.instance.empty_queues_cache_refresh_interval_sec` seconds
|
35
|
+
def self.setup!
|
36
|
+
refresher_timer_task = nil
|
37
|
+
|
38
|
+
Sidekiq.on(:startup) do
|
39
|
+
refresher_timer_task&.shutdown
|
40
|
+
refresher_timer_task = RefreshTimerTask.setup!(self)
|
41
|
+
end
|
42
|
+
|
43
|
+
Sidekiq.on(:shutdown) { refresher_timer_task&.shutdown }
|
44
|
+
end
|
45
|
+
|
46
|
+
# Attempts to update the global cache of empty queues by first acquiring a global lock
|
47
|
+
# If the lock is acquired, it brute force generates an accurate list of currently empty queues and
|
48
|
+
# then writes the updated list to the global cache
|
49
|
+
# The local queue cache is always updated as a result of this operation, either by using the recently generated
|
50
|
+
# list or fetching the most recent list from the global cache
|
51
|
+
#
|
52
|
+
# @return [Boolean] true if local cache was updated
|
53
|
+
def refresh!
|
54
|
+
return false unless local_lock.try_lock
|
55
|
+
|
56
|
+
begin
|
57
|
+
refresh_global_cache! || refresh_local_cache
|
58
|
+
ensure
|
59
|
+
local_lock.unlock
|
60
|
+
end
|
61
|
+
rescue => e
|
62
|
+
Sidekiq.logger.error { "Empty queues cache update failed: #{e}" }
|
63
|
+
raise
|
64
|
+
end
|
65
|
+
|
66
|
+
private
|
67
|
+
|
68
|
+
# Automatically updates local cache if global cache was updated
|
69
|
+
# @return [Boolean] true if cache was updated
|
70
|
+
def refresh_global_cache!
|
71
|
+
Sidekiq.logger.debug { "Refreshing global cache" }
|
72
|
+
|
73
|
+
global_lock do
|
74
|
+
Sidekiq.redis do |redis|
|
75
|
+
empty_queues = generate_empty_queues(redis)
|
76
|
+
|
77
|
+
update_global_cache(redis, empty_queues)
|
78
|
+
update_local_cache(empty_queues)
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
def generate_empty_queues(redis)
|
84
|
+
# Cursor is not atomic, so there may be duplicates because of concurrent update operations
|
85
|
+
queues = Sidekiq.redis { |r| r.sscan_each("queues").to_a.uniq }
|
86
|
+
|
87
|
+
queues_statuses =
|
88
|
+
redis.pipelined do |p|
|
89
|
+
queues.each do |queue|
|
90
|
+
p.exists?(QueueName.new(queue).pending)
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
queues.zip(queues_statuses).reject { |(_, exists)| exists }.map(&:first)
|
95
|
+
end
|
96
|
+
|
97
|
+
def refresh_local_cache
|
98
|
+
Sidekiq.logger.debug { "Refreshing local cache" }
|
99
|
+
|
100
|
+
# Cursor is not atomic, so there may be duplicates because of concurrent update operations
|
101
|
+
list = Sidekiq.redis { |redis| redis.sscan_each(KEY).to_a.uniq }
|
102
|
+
update_local_cache(list)
|
103
|
+
end
|
104
|
+
|
105
|
+
def update_global_cache(redis, list)
|
106
|
+
Sidekiq.logger.debug { "Setting global cache: #{list}" }
|
107
|
+
|
108
|
+
redis.multi do |multi|
|
109
|
+
multi.del(KEY)
|
110
|
+
multi.sadd(KEY, list) if list.any?
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
def update_local_cache(list)
|
115
|
+
Sidekiq.logger.debug { "Setting local cache: #{list}" }
|
116
|
+
|
117
|
+
@queues = list
|
118
|
+
end
|
119
|
+
|
120
|
+
# @return [Boolean] true if lock was acquired
|
121
|
+
def global_lock
|
122
|
+
Sidekiq.redis do |redis|
|
123
|
+
break false if skip_update?(redis) # Cheap check since lock will not be free most of the time
|
124
|
+
|
125
|
+
Redlock::Client.new([redis], :retry_count => 0).lock(LOCK_KEY, 30_000) do |locked|
|
126
|
+
break false unless locked
|
127
|
+
break false if skip_update?(redis)
|
128
|
+
|
129
|
+
yield
|
130
|
+
|
131
|
+
redis.set(LAST_RUN_KEY, redis.time.first)
|
132
|
+
end
|
133
|
+
end
|
134
|
+
end
|
135
|
+
|
136
|
+
def skip_update?(redis)
|
137
|
+
results = redis.pipelined { |pipeline| [pipeline.time, pipeline.get(LAST_RUN_KEY)] }
|
138
|
+
last_run_distance = results[0][0] - results[1].to_i
|
139
|
+
|
140
|
+
last_run_distance < Sidekiq::Ultimate::Configuration.instance.empty_queues_cache_refresh_interval_sec
|
141
|
+
end
|
142
|
+
end
|
143
|
+
end
|
144
|
+
end
|
@@ -2,8 +2,6 @@
|
|
2
2
|
|
3
3
|
require "monitor"
|
4
4
|
|
5
|
-
require "concurrent/utility/monotonic_time"
|
6
|
-
|
7
5
|
module Sidekiq
|
8
6
|
module Ultimate
|
9
7
|
# List that tracks when elements were added and enumerates over those not
|
@@ -24,7 +22,6 @@ module Sidekiq
|
|
24
22
|
# It does not deduplicates elements. Eviction happens only upon elements
|
25
23
|
# retrieval (see {#each}).
|
26
24
|
#
|
27
|
-
# @see http://ruby-concurrency.github.io/concurrent-ruby/Concurrent.html#monotonic_time-class_method
|
28
25
|
# @see https://ruby-doc.org/core/Process.html#method-c-clock_gettime
|
29
26
|
# @see https://linux.die.net/man/3/clock_gettime
|
30
27
|
#
|
@@ -52,7 +49,7 @@ module Sidekiq
|
|
52
49
|
# @return [ExpirableSet] self
|
53
50
|
def add(element, ttl:)
|
54
51
|
@mon.synchronize do
|
55
|
-
expires_at =
|
52
|
+
expires_at = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + ttl
|
56
53
|
|
57
54
|
# do not allow decrease element's expiry
|
58
55
|
@set[element] = expires_at if @set[element] < expires_at
|
@@ -71,7 +68,7 @@ module Sidekiq
|
|
71
68
|
return to_enum __method__ unless block_given?
|
72
69
|
|
73
70
|
@mon.synchronize do
|
74
|
-
horizon =
|
71
|
+
horizon = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
75
72
|
@set.each { |k, v| yield(k) if horizon <= v }
|
76
73
|
end
|
77
74
|
|