atomic-sidekiq 1.1.4 → 1.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +3 -0
- data/README.md +6 -3
- data/atomic-sidekiq.gemspec +2 -2
- data/lib/atomic_sidekiq/atomic_fetch.rb +12 -3
- data/lib/atomic_sidekiq/atomic_operation/expire.rb +12 -2
- data/lib/atomic_sidekiq/atomic_operation/heartbeat.rb +2 -0
- data/lib/atomic_sidekiq/atomic_operation/lua_scripts/expire.lua +4 -1
- data/lib/atomic_sidekiq/dead_job_collector.rb +8 -6
- data/lib/atomic_sidekiq/recovered_stats.rb +0 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: aeb25fc1520ae4298c7643e5e012bd08afa767332db6422f1cbf5a9b2e2569af
|
4
|
+
data.tar.gz: d4588254c282b2e07c9c35858c351436d848be87d2b94b2bf295b489f71b4f5b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ced404ab64bc0598c8bcabbcf9e75268f4351e0900de85468e655178f4cb0f2cda9481f93725529fbccfdc6d25cd1c9d66052a16f5f54b0ccda496e85c1deee6
|
7
|
+
data.tar.gz: 05cdd96f7a3ab054a8a55128913738ed8baa2ef3377d5a85b6fa87d1422322d92a1b7a80ccae96bf6fcb3e4ec31026ebdb516373cbede0a98dfbfc442966753d
|
data/.rubocop.yml
CHANGED
data/README.md
CHANGED
@@ -21,12 +21,15 @@ end
|
|
21
21
|
```
|
22
22
|
|
23
23
|
## Configuration
|
24
|
-
By default, jobs will expire and be re-queued after 1 hour if not acknowledged, and the "Collector" will check if for expired jobs every 60 seconds. This can be reconfigured as desired: _(Note that collection adds some overhead)_
|
24
|
+
By default, jobs will expire and be re-queued after 1 hour if not acknowledged, and the "Collector" will check if for expired jobs every 60 seconds. This can be reconfigured as desired: _(Note that collection adds some overhead)_. You
|
25
|
+
may also configure which queues shouldn't be recovered using the property `ignored_queues`. Jobs in those queues will
|
26
|
+
be expired without being pushed into the queue again.
|
25
27
|
```ruby
|
26
28
|
Sidekiq.configure_server do |config|
|
27
29
|
config.atomic_fetch!({
|
28
|
-
collection_interval: 5,
|
29
|
-
expiration_time: 1800
|
30
|
+
collection_interval: 5, # Unit: seconds
|
31
|
+
expiration_time: 1800, # Unit: seconds (30 minutes)
|
32
|
+
ignored_queues: ['cron'],
|
30
33
|
})
|
31
34
|
end
|
32
35
|
```
|
data/atomic-sidekiq.gemspec
CHANGED
@@ -2,8 +2,8 @@
|
|
2
2
|
|
3
3
|
Gem::Specification.new do |s|
|
4
4
|
s.name = "atomic-sidekiq"
|
5
|
-
s.version = "1.
|
6
|
-
s.date = "2018-12-
|
5
|
+
s.version = "1.2.0"
|
6
|
+
s.date = "2018-12-20"
|
7
7
|
s.summary = "Reliable fetcher for Sidekiq"
|
8
8
|
s.description = "Reliable fetcher for Sidekiq"
|
9
9
|
s.homepage = "https://github.com/Colex/atomic-sidekiq"
|
@@ -25,14 +25,16 @@ module AtomicSidekiq
|
|
25
25
|
collect_dead_jobs!
|
26
26
|
work = retrieve_op.perform(ordered_queues, expire_at)
|
27
27
|
return UnitOfWork.new(*work, in_flight_keymaker: keymaker) if work
|
28
|
+
|
28
29
|
sleep(poll_interval)
|
29
30
|
nil
|
30
31
|
end
|
31
32
|
|
32
33
|
private
|
33
34
|
|
34
|
-
attr_reader :retrieve_op, :queues, :
|
35
|
-
:
|
35
|
+
attr_reader :retrieve_op, :queues, :ignored_queues,
|
36
|
+
:strictly_ordered_queues, :collection_interval,
|
37
|
+
:poll_interval, :expiration_time,
|
36
38
|
:keymaker
|
37
39
|
|
38
40
|
def configure_atomic_fetch(options)
|
@@ -40,6 +42,8 @@ module AtomicSidekiq
|
|
40
42
|
@collection_interval = options[:collection_wait_time] ||
|
41
43
|
DEFAULT_COLLECTION_INTERVAL
|
42
44
|
@poll_interval = options[:poll_interval] || DEFAULT_POLL_INTERVAL
|
45
|
+
@ignored_queues = (options[:ignored_queues] || [])
|
46
|
+
.map { |q| "queue:#{q}" }
|
43
47
|
end
|
44
48
|
|
45
49
|
def ordered_queues
|
@@ -52,8 +56,13 @@ module AtomicSidekiq
|
|
52
56
|
|
53
57
|
def collect_dead_jobs!
|
54
58
|
return if @@next_collection > Time.now
|
59
|
+
|
55
60
|
@@next_collection = Time.now + collection_interval
|
56
|
-
DeadJobCollector.collect!(
|
61
|
+
DeadJobCollector.collect!(
|
62
|
+
ordered_queues,
|
63
|
+
in_flight_keymaker: keymaker,
|
64
|
+
skip_recovery_queues: ignored_queues
|
65
|
+
)
|
57
66
|
end
|
58
67
|
|
59
68
|
def expire_at
|
@@ -10,9 +10,19 @@ module AtomicSidekiq
|
|
10
10
|
super(in_flight_keymaker: nil)
|
11
11
|
end
|
12
12
|
|
13
|
-
def perform(queue, in_flight_key)
|
13
|
+
def perform(queue, in_flight_key, recover:)
|
14
14
|
redis do |conn|
|
15
|
-
conn.eval(
|
15
|
+
conn.eval(
|
16
|
+
EXPIRE_SCRIPT,
|
17
|
+
[
|
18
|
+
queue, # Queue Name
|
19
|
+
in_flight_key, # Key of the inflight job being expired
|
20
|
+
],
|
21
|
+
[
|
22
|
+
Time.now.utc.to_i, # Current time
|
23
|
+
recover, # Boolean flag: should it be recovered if expired
|
24
|
+
]
|
25
|
+
)
|
16
26
|
end
|
17
27
|
end
|
18
28
|
end
|
@@ -9,6 +9,7 @@ module AtomicSidekiq
|
|
9
9
|
def perform(jid:, timeout:)
|
10
10
|
key = in_flight_job_key(jid)
|
11
11
|
return unless key
|
12
|
+
|
12
13
|
redis do |conn|
|
13
14
|
conn.eval(HEARTBEAT_SCRIPT, [key], [expiration_date(timeout)])
|
14
15
|
end
|
@@ -26,6 +27,7 @@ module AtomicSidekiq
|
|
26
27
|
loop do
|
27
28
|
it, keys = redis { |conn| conn.scan(it, match: matcher) }
|
28
29
|
return keys[0] if keys.count > 0
|
30
|
+
|
29
31
|
it = it.to_i
|
30
32
|
return if it.zero?
|
31
33
|
end
|
@@ -1,6 +1,7 @@
|
|
1
1
|
local queue = KEYS[1]
|
2
2
|
local in_flight_key = KEYS[2]
|
3
3
|
local now = tonumber(ARGV[1])
|
4
|
+
local recover = ARGV[2] == 'true'
|
4
5
|
|
5
6
|
local job = redis.call('get', in_flight_key)
|
6
7
|
if (not job) then return nil end
|
@@ -9,7 +10,9 @@ local expiration = tonumber(string.match(job, '"expire_at":([0-9]*)'))
|
|
9
10
|
if expiration > now then return nil end
|
10
11
|
job = string.gsub(job, ',?"expire_at":[0-9]*', '')
|
11
12
|
|
12
|
-
|
13
|
+
if recover then
|
14
|
+
redis.call('lpush', queue, job)
|
15
|
+
end
|
13
16
|
redis.call('del', in_flight_key)
|
14
17
|
|
15
18
|
return { queue, job }
|
@@ -1,9 +1,10 @@
|
|
1
1
|
module AtomicSidekiq
|
2
2
|
class DeadJobCollector
|
3
3
|
class << self
|
4
|
-
def collect!(queues, in_flight_keymaker:)
|
4
|
+
def collect!(queues, in_flight_keymaker:, skip_recovery_queues: [])
|
5
5
|
queues.each do |q|
|
6
|
-
new(q, in_flight_keymaker: in_flight_keymaker)
|
6
|
+
new(q, in_flight_keymaker: in_flight_keymaker)
|
7
|
+
.collect!(skip_recovery: skip_recovery_queues.include?(q))
|
7
8
|
end
|
8
9
|
end
|
9
10
|
end
|
@@ -15,17 +16,18 @@ module AtomicSidekiq
|
|
15
16
|
@expire_op = AtomicOperation::Expire.new
|
16
17
|
end
|
17
18
|
|
18
|
-
def collect!
|
19
|
-
each_keys { |job_key| expire!(job_key) }
|
19
|
+
def collect!(skip_recovery: false)
|
20
|
+
each_keys { |job_key| expire!(job_key, skip_recovery: skip_recovery) }
|
20
21
|
end
|
21
22
|
|
22
23
|
private
|
23
24
|
|
24
25
|
attr_reader :queue, :in_flight_keymaker, :expire_op, :recovered_stats
|
25
26
|
|
26
|
-
def expire!(job_key)
|
27
|
-
recovered = expire_op.perform(queue, job_key)
|
27
|
+
def expire!(job_key, skip_recovery:)
|
28
|
+
recovered = expire_op.perform(queue, job_key, recover: !skip_recovery)
|
28
29
|
return if recovered.nil?
|
30
|
+
|
29
31
|
job = JSON.parse(recovered[1])
|
30
32
|
recovered_stats.increment!(job)
|
31
33
|
job
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: atomic-sidekiq
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Alex Correia Santos
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-12-
|
11
|
+
date: 2018-12-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|