sidekiq-robust-job 0.2.0 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Changelog.md +6 -0
- data/Gemfile.lock +27 -27
- data/README.md +9 -3
- data/lib/sidekiq/robust/job/version.rb +1 -1
- data/lib/sidekiq_robust_job/enqueue_conflict_resolution_strategy/drop_self.rb +1 -1
- data/lib/sidekiq_robust_job/enqueue_conflict_resolution_strategy/replace.rb +1 -1
- data/lib/sidekiq_robust_job/model.rb +1 -0
- data/lib/sidekiq_robust_job/repository.rb +17 -0
- data/lib/sidekiq_robust_job/uniqueness_strategy/while_executing.rb +15 -0
- data/lib/sidekiq_robust_job/uniqueness_strategy.rb +11 -0
- data/lib/sidekiq_robust_job.rb +1 -0
- metadata +4 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 16456089ded6f826bcb4d0c9e1c2538dbf577a2204b512204f7228553cfe2405
|
4
|
+
data.tar.gz: 96b63fb47b131378d8409c3e0022b78749e8ca9f231ceb469fdd8a21af282416
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c78e1907f15a9c881273c4e8f26a0937bd8233267e358aeb374fe91a5b68db28191e9a863a7c7761e5c6ff0994dd6c3ae57cf94c191f93b81b2a1ee88e24715e
|
7
|
+
data.tar.gz: a901ad322c8e4a370ea2bf84e6ceb42c44930c1cd9804c0a23ad2e9230979b8ede1b13466e3dc38561bbe3c79a7d48be57aa40991044bde1f2ca6f8fbff8b0b1
|
data/Changelog.md
CHANGED
@@ -1,5 +1,11 @@
|
|
1
1
|
# Changelog
|
2
2
|
|
3
|
+
## master
|
4
|
+
|
5
|
+
## 0.3.0
|
6
|
+
- Fix `enqueue_conflict_resolution_strategies` to work well with `until_executing` strategy. Handles corner case when job is being processed, yet we can't enqueue another one
|
7
|
+
- Introduce `while_executing` uniqueness strategy, which is basically execution mutex - ensures only that 2 jobs are not executed in parallel, without considering enqueueing uniqueness.
|
8
|
+
|
3
9
|
## 0.2.0
|
4
10
|
- Introduce `persist_self_dropped_jobs` config option
|
5
11
|
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
sidekiq-robust-job (0.
|
4
|
+
sidekiq-robust-job (0.3.0)
|
5
5
|
activesupport (>= 5)
|
6
6
|
sidekiq (>= 5)
|
7
7
|
sidekiq-cron (~> 1)
|
@@ -9,55 +9,55 @@ PATH
|
|
9
9
|
GEM
|
10
10
|
remote: https://rubygems.org/
|
11
11
|
specs:
|
12
|
-
actionpack (5.2.4.
|
13
|
-
actionview (= 5.2.4.
|
14
|
-
activesupport (= 5.2.4.
|
12
|
+
actionpack (5.2.4.5)
|
13
|
+
actionview (= 5.2.4.5)
|
14
|
+
activesupport (= 5.2.4.5)
|
15
15
|
rack (~> 2.0, >= 2.0.8)
|
16
16
|
rack-test (>= 0.6.3)
|
17
17
|
rails-dom-testing (~> 2.0)
|
18
18
|
rails-html-sanitizer (~> 1.0, >= 1.0.2)
|
19
|
-
actionview (5.2.4.
|
20
|
-
activesupport (= 5.2.4.
|
19
|
+
actionview (5.2.4.5)
|
20
|
+
activesupport (= 5.2.4.5)
|
21
21
|
builder (~> 3.1)
|
22
22
|
erubi (~> 1.4)
|
23
23
|
rails-dom-testing (~> 2.0)
|
24
24
|
rails-html-sanitizer (~> 1.0, >= 1.0.3)
|
25
|
-
activemodel (5.2.4.
|
26
|
-
activesupport (= 5.2.4.
|
27
|
-
activerecord (5.2.4.
|
28
|
-
activemodel (= 5.2.4.
|
29
|
-
activesupport (= 5.2.4.
|
25
|
+
activemodel (5.2.4.5)
|
26
|
+
activesupport (= 5.2.4.5)
|
27
|
+
activerecord (5.2.4.5)
|
28
|
+
activemodel (= 5.2.4.5)
|
29
|
+
activesupport (= 5.2.4.5)
|
30
30
|
arel (>= 9.0)
|
31
|
-
activesupport (5.2.4.
|
31
|
+
activesupport (5.2.4.5)
|
32
32
|
concurrent-ruby (~> 1.0, >= 1.0.2)
|
33
33
|
i18n (>= 0.7, < 2)
|
34
34
|
minitest (~> 5.1)
|
35
35
|
tzinfo (~> 1.1)
|
36
36
|
arel (9.0.0)
|
37
37
|
builder (3.2.4)
|
38
|
-
concurrent-ruby (1.1.
|
38
|
+
concurrent-ruby (1.1.9)
|
39
39
|
connection_pool (2.2.3)
|
40
40
|
crass (1.0.6)
|
41
41
|
diff-lcs (1.4.4)
|
42
42
|
erubi (1.10.0)
|
43
|
-
et-orbi (1.2.
|
43
|
+
et-orbi (1.2.7)
|
44
44
|
tzinfo
|
45
45
|
factory_bot (6.1.0)
|
46
46
|
activesupport (>= 5.0.0)
|
47
47
|
factory_bot_rails (6.1.0)
|
48
48
|
factory_bot (~> 6.1.0)
|
49
49
|
railties (>= 5.0.0)
|
50
|
-
fugit (1.
|
51
|
-
et-orbi (~> 1
|
50
|
+
fugit (1.5.3)
|
51
|
+
et-orbi (~> 1, >= 1.2.7)
|
52
52
|
raabro (~> 1.4)
|
53
|
-
i18n (1.8.
|
53
|
+
i18n (1.8.10)
|
54
54
|
concurrent-ruby (~> 1.0)
|
55
|
-
loofah (2.
|
55
|
+
loofah (2.10.0)
|
56
56
|
crass (~> 1.0.2)
|
57
57
|
nokogiri (>= 1.5.9)
|
58
58
|
method_source (1.0.0)
|
59
59
|
mini_portile2 (2.4.0)
|
60
|
-
minitest (5.14.
|
60
|
+
minitest (5.14.4)
|
61
61
|
nokogiri (1.10.10)
|
62
62
|
mini_portile2 (~> 2.4.0)
|
63
63
|
pg (1.2.3)
|
@@ -70,9 +70,9 @@ GEM
|
|
70
70
|
nokogiri (>= 1.6)
|
71
71
|
rails-html-sanitizer (1.3.0)
|
72
72
|
loofah (~> 2.3)
|
73
|
-
railties (5.2.4.
|
74
|
-
actionpack (= 5.2.4.
|
75
|
-
activesupport (= 5.2.4.
|
73
|
+
railties (5.2.4.5)
|
74
|
+
actionpack (= 5.2.4.5)
|
75
|
+
activesupport (= 5.2.4.5)
|
76
76
|
method_source
|
77
77
|
rake (>= 0.8.7)
|
78
78
|
thor (>= 0.19.0, < 2.0)
|
@@ -100,13 +100,13 @@ GEM
|
|
100
100
|
connection_pool (>= 2.2.2)
|
101
101
|
rack (~> 2.0)
|
102
102
|
redis (>= 4.2.0)
|
103
|
-
sidekiq-cron (1.
|
104
|
-
fugit (
|
103
|
+
sidekiq-cron (1.3.0)
|
104
|
+
fugit (>= 1.1)
|
105
105
|
sidekiq (>= 4.2.1)
|
106
|
-
thor (1.0
|
106
|
+
thor (1.1.0)
|
107
107
|
thread_safe (0.3.6)
|
108
108
|
timecop (0.9.2)
|
109
|
-
tzinfo (1.2.
|
109
|
+
tzinfo (1.2.9)
|
110
110
|
thread_safe (~> 0.1)
|
111
111
|
|
112
112
|
PLATFORMS
|
@@ -124,4 +124,4 @@ DEPENDENCIES
|
|
124
124
|
timecop
|
125
125
|
|
126
126
|
BUNDLED WITH
|
127
|
-
2.
|
127
|
+
2.2.22
|
data/README.md
CHANGED
@@ -145,7 +145,7 @@ Although keep in mind that using this feature comes with some performance penalt
|
|
145
145
|
|
146
146
|
If you have a lot of conflicts within a short period, consider using `perform_in` instead of `perform_async` and add some random number of seconds (ideally, below 1 minute) to make it easier to apply enqueue conflict resolution strategy.
|
147
147
|
|
148
|
-
If you enqueue a lot of the same jobs (same class, same arguments) in a short period of time and `drop_self` strategy, you should consider setting `persist_self_dropped_jobs` config option to false. By default, it's true which means that even the jobs that are dropped are persisted, which might be useful for some profiling or even figuring out in the first place that you have an issue like this. However, under such circumstances this is likely to result in heavier queries fetching a lot of rows from the database, causing a high database load.
|
148
|
+
If you enqueue a lot of the same jobs (same class, same arguments) in a short period of time and `drop_self` strategy, you should consider setting `persist_self_dropped_jobs` config option to false. By default, it's true which means that even the jobs that are dropped are persisted, which might be useful for some profiling or even figuring out in the first place that you have an issue like this. However, under such circumstances this is likely to result in heavier queries fetching a lot of rows from the database, causing a high database load.
|
149
149
|
|
150
150
|
Here is an example how to use it:
|
151
151
|
|
@@ -155,8 +155,7 @@ class MyJob
|
|
155
155
|
include Sidekiq::Worker
|
156
156
|
include SidekiqRobustJob::SidekiqJobExtensions
|
157
157
|
|
158
|
-
sidekiq_options queue: "critical", enqueue_conflict_resolution_strategy: "drop_self",
|
159
|
-
persist_self_dropped_jobs: false
|
158
|
+
sidekiq_options queue: "critical", enqueue_conflict_resolution_strategy: "drop_self", persist_self_dropped_jobs: false
|
160
159
|
|
161
160
|
def call(user_id)
|
162
161
|
User.find(user_id).do_something
|
@@ -164,6 +163,12 @@ class MyJob
|
|
164
163
|
end
|
165
164
|
```
|
166
165
|
|
166
|
+
To make it more likely for self-dropping to kick-in, especially if there is some sort of burst of multiple same jobs being applied at almost the time, consider adding some delay for the execution of the job:
|
167
|
+
|
168
|
+
```
|
169
|
+
MyJob.perform_in((5..30).to_a.sample.seconds, user_id)
|
170
|
+
```
|
171
|
+
|
167
172
|
#### Execution Uniqueness (Mutex)
|
168
173
|
|
169
174
|
This feature is about handling a "conflict" (determined by a digest generated based on the job class and its arguments) when there is already the "same job" getting executed (i.e. same job class and arguments) at the same time.
|
@@ -173,6 +178,7 @@ Let's say that there is already a job scheduled to be executed just in a moment
|
|
173
178
|
1. `no_uniqueness` - this is a default when you don't specify anything. In this case, both jobs will be executed.
|
174
179
|
2. `until_executed` - One of the jobs acquires mutex using Redlock. When job is finished, it drops other pending jobs (and assigns `dropped_by_job_id` equal to the job that acquired the lock) with the same digest (based on job's class and arguments), and releases the lock. The job that failed to acquire a mutex is rescheduled (not dropped though, just to be on the safe side) and will be executed in the interval determined by `reschedule_interval_in_seconds` (5 seconds by default).
|
175
180
|
3. `until_executing` - One of the jobs acquires mutex using Redlock, it drops and assigns `dropped_by_job_id` equal to the job that acquired the lock) other pending jobs with the same digest (based on job's class and arguments) and releases the lock. And then it executes the actual logic behind the job. The job that failed to acquire a lock is rescheduled (not dropped though, just to be on the safe side) and will be executed in the interval determined by `reschedule_interval_in_seconds` (5 seconds by default).
|
181
|
+
4. `while_executing` - One of the jobs acquires mutex using Redlock. The other job that failed to acquire a lock is rescheduled (not dropped though, just to be on the safe side) and will be executed in the interval determined by `reschedule_interval_in_seconds` (5 seconds by default). This strategy grants you only execution mutex, unlike `until_executed` or `until_executing` it will not drop any enqueued jobs.
|
176
182
|
|
177
183
|
If you want to use this feature, declare in with other Sidekiq options in the job:
|
178
184
|
|
@@ -2,7 +2,7 @@ class SidekiqRobustJob
|
|
2
2
|
class EnqueueConflictResolutionStrategy
|
3
3
|
class DropSelf < SidekiqRobustJob::EnqueueConflictResolutionStrategy::Base
|
4
4
|
def execute(job)
|
5
|
-
if jobs_repository.
|
5
|
+
if jobs_repository.not_started_for_digest(job.digest, exclude_id: job.id).any?
|
6
6
|
job.drop(dropped_by_job_id: job.id, clock: clock)
|
7
7
|
end
|
8
8
|
end
|
@@ -2,7 +2,7 @@ class SidekiqRobustJob
|
|
2
2
|
class EnqueueConflictResolutionStrategy
|
3
3
|
class Replace < SidekiqRobustJob::EnqueueConflictResolutionStrategy::Base
|
4
4
|
def execute(job)
|
5
|
-
jobs_repository.
|
5
|
+
jobs_repository.drop_not_started_jobs_by_digest(
|
6
6
|
dropped_by_job_id: job.id,
|
7
7
|
digest: job.digest,
|
8
8
|
exclude_id: job.id
|
@@ -9,6 +9,7 @@ class SidekiqRobustJob
|
|
9
9
|
SidekiqRobustJob::UniquenessStrategy.no_uniqueness,
|
10
10
|
SidekiqRobustJob::UniquenessStrategy.until_executing,
|
11
11
|
SidekiqRobustJob::UniquenessStrategy.until_executed,
|
12
|
+
SidekiqRobustJob::UniquenessStrategy.while_executing,
|
12
13
|
].map(&:to_s)
|
13
14
|
}, on: :create
|
14
15
|
|
@@ -34,6 +34,14 @@ class SidekiqRobustJob
|
|
34
34
|
.select { |potentially_missed_job| missed_job_policy.call(potentially_missed_job) }
|
35
35
|
end
|
36
36
|
|
37
|
+
def not_started_for_digest(digest, exclude_id:)
|
38
|
+
jobs_database
|
39
|
+
.where(digest: digest)
|
40
|
+
.where(started_at: nil)
|
41
|
+
.where(dropped_at: nil)
|
42
|
+
.where.not(id: exclude_id)
|
43
|
+
end
|
44
|
+
|
37
45
|
def unprocessed_for_digest(digest, exclude_id:)
|
38
46
|
jobs_database
|
39
47
|
.where(digest: digest)
|
@@ -50,5 +58,14 @@ class SidekiqRobustJob
|
|
50
58
|
end
|
51
59
|
end
|
52
60
|
end
|
61
|
+
|
62
|
+
def drop_not_started_jobs_by_digest(dropped_by_job_id:, digest:, exclude_id:)
|
63
|
+
transaction do
|
64
|
+
not_started_for_digest(digest, exclude_id: exclude_id).lock!.find_each do |job|
|
65
|
+
job.drop(dropped_by_job_id: dropped_by_job_id)
|
66
|
+
save(job)
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
53
70
|
end
|
54
71
|
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
class SidekiqRobustJob
|
2
|
+
class UniquenessStrategy
|
3
|
+
class WhileExecuting < SidekiqRobustJob::UniquenessStrategy::Base
|
4
|
+
def execute(job)
|
5
|
+
lock(job) do |locked|
|
6
|
+
if locked
|
7
|
+
perform_job_and_finalize(job)
|
8
|
+
else
|
9
|
+
job.reschedule and return
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -12,6 +12,10 @@ class SidekiqRobustJob
|
|
12
12
|
:until_executed
|
13
13
|
end
|
14
14
|
|
15
|
+
def self.while_executing
|
16
|
+
:while_executing
|
17
|
+
end
|
18
|
+
|
15
19
|
attr_reader :locker, :lock_ttl_proc, :jobs_repository, :memory_monitor
|
16
20
|
private :locker, :lock_ttl_proc, :jobs_repository, :memory_monitor
|
17
21
|
|
@@ -45,6 +49,13 @@ class SidekiqRobustJob
|
|
45
49
|
jobs_repository: jobs_repository,
|
46
50
|
memory_monitor: memory_monitor
|
47
51
|
)
|
52
|
+
when SidekiqRobustJob::UniquenessStrategy.while_executing
|
53
|
+
SidekiqRobustJob::UniquenessStrategy::WhileExecuting.new(
|
54
|
+
locker: locker,
|
55
|
+
lock_ttl_proc: lock_ttl_proc,
|
56
|
+
jobs_repository: jobs_repository,
|
57
|
+
memory_monitor: memory_monitor
|
58
|
+
)
|
48
59
|
else
|
49
60
|
raise UnknownStrategyError.new(strategy)
|
50
61
|
end
|
data/lib/sidekiq_robust_job.rb
CHANGED
@@ -20,6 +20,7 @@ require "sidekiq_robust_job/uniqueness_strategy/base"
|
|
20
20
|
require "sidekiq_robust_job/uniqueness_strategy/no_uniqueness"
|
21
21
|
require "sidekiq_robust_job/uniqueness_strategy/until_executed"
|
22
22
|
require "sidekiq_robust_job/uniqueness_strategy/until_executing"
|
23
|
+
require "sidekiq_robust_job/uniqueness_strategy/while_executing"
|
23
24
|
require "sidekiq/cron/job"
|
24
25
|
require "sidekiq"
|
25
26
|
require "active_support/concern"
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sidekiq-robust-job
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Karol Galanciak
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-04-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: sidekiq
|
@@ -208,6 +208,7 @@ files:
|
|
208
208
|
- lib/sidekiq_robust_job/uniqueness_strategy/no_uniqueness.rb
|
209
209
|
- lib/sidekiq_robust_job/uniqueness_strategy/until_executed.rb
|
210
210
|
- lib/sidekiq_robust_job/uniqueness_strategy/until_executing.rb
|
211
|
+
- lib/sidekiq_robust_job/uniqueness_strategy/while_executing.rb
|
211
212
|
- sidekiq-robust-job.gemspec
|
212
213
|
homepage: https://github.com/BookingSync/sidekiq-robust-job
|
213
214
|
licenses:
|
@@ -231,7 +232,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
231
232
|
- !ruby/object:Gem::Version
|
232
233
|
version: '0'
|
233
234
|
requirements: []
|
234
|
-
rubygems_version: 3.
|
235
|
+
rubygems_version: 3.2.22
|
235
236
|
signing_key:
|
236
237
|
specification_version: 4
|
237
238
|
summary: A gem making Sidekiq jobs fully durable with some extra features
|