sidekiq-max-jobs 0.0.4 → 0.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +3 -3
- data/.rubocop_todo.yml +9 -4
- data/README.md +13 -1
- data/VERSION +1 -1
- data/lib/sidekiq/middleware/server/max_jobs.rb +107 -40
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9bf624b8b84bbfd78039b22826e440c7218c9850207a55a1a24bfc046a0812a4
|
4
|
+
data.tar.gz: 364ca8c57ac52151636f7c9a0feb7ffe18d758fd95772fb692b0802fb634bef6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 12548204b50a4f9d3dfec6470d3488b03fc672c7c41c12e6554ed350004ce63b42d322043c21dfc023fbd6fc819a724c5a167d62494a0615be0c8d4e71bfda19
|
7
|
+
data.tar.gz: 7e704fde12c9f64b200ac001f6824821ec6e0698235d4493e01a30ff49f27e3742373ae38f0ef91c013b5ee08986bf619cb6c2922eff44c79b3898444b4c3984
|
data/.rubocop.yml
CHANGED
@@ -12,6 +12,9 @@ AllCops:
|
|
12
12
|
Layout/DotPosition:
|
13
13
|
Enabled: false
|
14
14
|
|
15
|
+
Layout/LineLength:
|
16
|
+
Max: 120
|
17
|
+
|
15
18
|
Layout/SpaceInLambdaLiteral:
|
16
19
|
EnforcedStyle: require_space
|
17
20
|
|
@@ -19,9 +22,6 @@ Metrics/BlockLength:
|
|
19
22
|
Exclude:
|
20
23
|
- 'spec/**/*'
|
21
24
|
|
22
|
-
Metrics/LineLength:
|
23
|
-
Max: 120
|
24
|
-
|
25
25
|
Style/Documentation:
|
26
26
|
Enabled: false
|
27
27
|
|
data/.rubocop_todo.yml
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# This configuration was generated by
|
2
2
|
# `rubocop --auto-gen-config`
|
3
|
-
# on 2020-06-17
|
3
|
+
# on 2020-06-17 22:45:10 -0500 using RuboCop version 0.85.1.
|
4
4
|
# The point is for the user to remove these configuration records
|
5
5
|
# one by one as the offenses are removed from the code base.
|
6
6
|
# Note that changes in the inspected code, or installation of new
|
@@ -14,7 +14,7 @@ Lint/RescueException:
|
|
14
14
|
# Offense count: 1
|
15
15
|
# Configuration parameters: IgnoredMethods.
|
16
16
|
Metrics/AbcSize:
|
17
|
-
Max:
|
17
|
+
Max: 27
|
18
18
|
|
19
19
|
# Offense count: 1
|
20
20
|
# Configuration parameters: CountComments, ExcludedMethods.
|
@@ -25,12 +25,17 @@ Metrics/BlockLength:
|
|
25
25
|
# Offense count: 1
|
26
26
|
# Configuration parameters: CountComments.
|
27
27
|
Metrics/ClassLength:
|
28
|
-
Max:
|
28
|
+
Max: 152
|
29
|
+
|
30
|
+
# Offense count: 1
|
31
|
+
# Configuration parameters: IgnoredMethods.
|
32
|
+
Metrics/CyclomaticComplexity:
|
33
|
+
Max: 7
|
29
34
|
|
30
35
|
# Offense count: 1
|
31
36
|
# Configuration parameters: CountComments, ExcludedMethods.
|
32
37
|
Metrics/MethodLength:
|
33
|
-
Max:
|
38
|
+
Max: 24
|
34
39
|
|
35
40
|
# Offense count: 1
|
36
41
|
# Cop supports --auto-correct.
|
data/README.md
CHANGED
@@ -52,7 +52,7 @@ end
|
|
52
52
|
If everything above is successful the next time you start your worker you will
|
53
53
|
see a message like the following:
|
54
54
|
```bash
|
55
|
-
2020-06-10T00:23:31.789Z pid=73703 tid=oxifk6l13 INFO: Max-Jobs middleware enabled, shutting down pid: 73703
|
55
|
+
2020-06-10T00:23:31.789Z pid=73703 tid=oxifk6l13 INFO: Max-Jobs middleware enabled, shutting down pid: 73703 when max-jobs quota is reached
|
56
56
|
```
|
57
57
|
|
58
58
|
Configuration Options
|
@@ -75,6 +75,18 @@ number between 1 and the value specified. This value is added to the
|
|
75
75
|
of your `Worker(s)` restart at / around the same time (default:
|
76
76
|
`rand((ENV['MAX_JOBS_JITTER'] || 1).to_i)`)
|
77
77
|
|
78
|
+
Important Note
|
79
|
+
--------------
|
80
|
+
|
81
|
+
When determining if the max-job quota has been reached the total jobs processed
|
82
|
+
is checked first, followed by the jobs processed for the current queue. If your
|
83
|
+
`Worker(s)` are handling multiple queues it is generally recommended that you
|
84
|
+
set the total value to the same value as your highest queue value (e.g. if you
|
85
|
+
had `MAX_JOBS_FOO=100` and `MAX_JOBS_BAR=200` it probably makes sense to set
|
86
|
+
`MAX_JOBS=200`, if not a little bit lower). Setting the right limits ultimately
|
87
|
+
depends on the intensity / resource needs of the work being performed. The same
|
88
|
+
rule of thumb applies to `MAX_JOBS_JITTER` as well.
|
89
|
+
|
78
90
|
Contributing
|
79
91
|
------------
|
80
92
|
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.0.
|
1
|
+
0.0.5
|
@@ -20,22 +20,40 @@ module Sidekiq
|
|
20
20
|
@cache ||= {}
|
21
21
|
end
|
22
22
|
|
23
|
-
def counter
|
24
|
-
key = counter_key
|
25
|
-
|
23
|
+
def counter
|
24
|
+
key = counter_key
|
25
|
+
cache[key] ||= 0
|
26
|
+
end
|
26
27
|
|
27
|
-
|
28
|
+
def counter_for_queue(queue)
|
29
|
+
key = counter_for_queue_key(queue)
|
30
|
+
cache[key] ||= 0
|
28
31
|
end
|
29
32
|
|
30
|
-
def
|
33
|
+
def counter_for_queue_key(queue)
|
31
34
|
"COUNTER_#{queue.upcase}"
|
32
35
|
end
|
33
36
|
|
34
|
-
def
|
35
|
-
|
36
|
-
|
37
|
+
def counter_key
|
38
|
+
'COUNTER'
|
39
|
+
end
|
40
|
+
|
41
|
+
def default_max_jobs
|
42
|
+
100
|
43
|
+
end
|
44
|
+
|
45
|
+
def default_max_jobs_jitter
|
46
|
+
1
|
47
|
+
end
|
48
|
+
|
49
|
+
def increment_counter!
|
50
|
+
key = counter_key
|
51
|
+
cache[key] = (cache[key] || 0).next
|
52
|
+
end
|
37
53
|
|
38
|
-
|
54
|
+
def increment_counter_for_queue!(queue)
|
55
|
+
key = counter_for_queue_key(queue)
|
56
|
+
cache[key] = (cache[key] || 0).next
|
39
57
|
end
|
40
58
|
|
41
59
|
def log_info(message)
|
@@ -43,65 +61,98 @@ module Sidekiq
|
|
43
61
|
end
|
44
62
|
|
45
63
|
def log_initialization!
|
46
|
-
log_info("Max-Jobs middleware enabled, shutting down pid: #{pid}
|
64
|
+
log_info("Max-Jobs middleware enabled, shutting down pid: #{pid} when max-jobs quota is reached")
|
47
65
|
end
|
48
66
|
|
49
|
-
def max_jobs
|
50
|
-
key = max_jobs_key
|
51
|
-
|
67
|
+
def max_jobs
|
68
|
+
key = max_jobs_key
|
69
|
+
cache[key] ||= (ENV[key] || default_max_jobs).to_i
|
70
|
+
end
|
52
71
|
|
53
|
-
|
72
|
+
def max_jobs_for_queue(queue)
|
73
|
+
key = max_jobs_for_queue_key(queue)
|
74
|
+
cache[key] ||= (
|
54
75
|
ENV[key] ||
|
55
|
-
ENV[
|
56
|
-
|
76
|
+
ENV[max_jobs_key] ||
|
77
|
+
default_max_jobs
|
57
78
|
).to_i
|
58
79
|
end
|
59
80
|
|
60
|
-
def
|
81
|
+
def max_jobs_for_queue_key(queue)
|
61
82
|
"MAX_JOBS_#{queue.upcase}"
|
62
83
|
end
|
63
84
|
|
64
|
-
def max_jobs_jitter
|
65
|
-
key = max_jobs_jitter_key
|
66
|
-
|
85
|
+
def max_jobs_jitter
|
86
|
+
key = max_jobs_jitter_key
|
87
|
+
cache[key] ||= rand((ENV[key] || default_max_jobs_jitter).to_i)
|
88
|
+
end
|
67
89
|
|
68
|
-
|
90
|
+
def max_jobs_jitter_for_queue(queue)
|
91
|
+
key = max_jobs_jitter_for_queue_key(queue)
|
92
|
+
cache[key] ||= rand(
|
69
93
|
(
|
70
94
|
ENV[key] ||
|
71
|
-
ENV[
|
72
|
-
|
95
|
+
ENV[max_jobs_jitter_key] ||
|
96
|
+
default_max_jobs_jitter
|
73
97
|
).to_i
|
74
98
|
)
|
75
99
|
end
|
76
100
|
|
77
|
-
def
|
101
|
+
def max_jobs_jitter_for_queue_key(queue)
|
78
102
|
"MAX_JOBS_JITTER_#{queue.upcase}"
|
79
103
|
end
|
80
104
|
|
81
|
-
def
|
82
|
-
|
83
|
-
|
105
|
+
def max_jobs_jitter_key
|
106
|
+
'MAX_JOBS_JITTER'
|
107
|
+
end
|
84
108
|
|
85
|
-
|
109
|
+
def max_jobs_key
|
110
|
+
'MAX_JOBS'
|
86
111
|
end
|
87
112
|
|
88
|
-
def
|
113
|
+
def max_jobs_with_jitter
|
114
|
+
key = max_jobs_with_jitter_key
|
115
|
+
cache[key] ||= (max_jobs + max_jobs_jitter)
|
116
|
+
end
|
117
|
+
|
118
|
+
def max_jobs_with_jitter_for_queue(queue)
|
119
|
+
key = max_jobs_with_jitter_for_queue_key(queue)
|
120
|
+
cache[key] ||= \
|
121
|
+
(max_jobs_for_queue(queue) + max_jobs_jitter_for_queue(queue))
|
122
|
+
end
|
123
|
+
|
124
|
+
def max_jobs_with_jitter_for_queue_key(queue)
|
89
125
|
"MAX_JOBS_WITH_JITTER_#{queue.upcase}"
|
90
126
|
end
|
91
127
|
|
92
|
-
def
|
93
|
-
|
94
|
-
|
128
|
+
def max_jobs_with_jitter_key
|
129
|
+
'MAX_JOBS_WITH_JITTER'
|
130
|
+
end
|
95
131
|
|
96
|
-
|
132
|
+
def mutex
|
133
|
+
key = mutex_key
|
134
|
+
cache[key] ||= ::Mutex.new
|
97
135
|
end
|
98
136
|
|
99
|
-
def mutex_key
|
100
|
-
|
137
|
+
def mutex_key
|
138
|
+
'MUTEX'
|
101
139
|
end
|
102
140
|
|
103
141
|
def pid
|
104
|
-
|
142
|
+
key = pid_key
|
143
|
+
cache[key] ||= ::Process.pid
|
144
|
+
end
|
145
|
+
|
146
|
+
def pid_key
|
147
|
+
'PID'
|
148
|
+
end
|
149
|
+
|
150
|
+
def quota_met?
|
151
|
+
counter == max_jobs_with_jitter
|
152
|
+
end
|
153
|
+
|
154
|
+
def quota_met_for_queue?(queue)
|
155
|
+
counter_for_queue(queue) == max_jobs_with_jitter_for_queue(queue)
|
105
156
|
end
|
106
157
|
end
|
107
158
|
|
@@ -121,13 +172,29 @@ module Sidekiq
|
|
121
172
|
raise
|
122
173
|
ensure
|
123
174
|
if !exception_raised
|
124
|
-
self.class.mutex
|
125
|
-
|
175
|
+
self.class.mutex.synchronize do
|
176
|
+
terminate = false
|
177
|
+
|
178
|
+
# Increment the total counter
|
179
|
+
self.class.increment_counter!
|
126
180
|
|
127
|
-
if
|
181
|
+
# First check if the total quota has been met
|
182
|
+
if self.class.quota_met?
|
128
183
|
self.class.log_info("Max-Jobs quota met, shutting down pid: #{self.class.pid}")
|
129
|
-
|
184
|
+
terminate = true
|
185
|
+
end
|
186
|
+
|
187
|
+
# Increment the queue specific counter
|
188
|
+
self.class.increment_counter_for_queue!(queue)
|
189
|
+
|
190
|
+
# Now check if the queue specific quota has been met
|
191
|
+
if !terminate && self.class.quota_met_for_queue?(queue)
|
192
|
+
self.class.log_info(%(Max-Jobs quota met for queue: "#{queue}", shutting down pid: #{self.class.pid}))
|
193
|
+
terminate = true
|
130
194
|
end
|
195
|
+
|
196
|
+
# If applicable, TERMinate the `Process`
|
197
|
+
::Process.kill('TERM', self.class.pid) if terminate
|
131
198
|
end
|
132
199
|
end
|
133
200
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sidekiq-max-jobs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jonathan W. Zaleski
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-06-
|
11
|
+
date: 2020-06-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: sidekiq
|