atomic-sidekiq 1.0.0 → 1.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +3 -0
- data/.rubocop.yml +28 -0
- data/.travis.yml +16 -0
- data/Gemfile +5 -0
- data/Gemfile.lock +16 -5
- data/README.md +19 -0
- data/atomic-sidekiq.gemspec +5 -5
- data/bin/sidekiqload +0 -1
- data/lib/atomic-sidekiq.rb +1 -1
- data/lib/atomic_sidekiq/atomic_fetch.rb +24 -15
- data/lib/atomic_sidekiq/atomic_operation/acknowledge.rb +2 -2
- data/lib/atomic_sidekiq/atomic_operation/base.rb +5 -10
- data/lib/atomic_sidekiq/atomic_operation/expire.rb +6 -5
- data/lib/atomic_sidekiq/atomic_operation/heartbeat.rb +35 -0
- data/lib/atomic_sidekiq/atomic_operation/lua_scripts/heartbeat.lua +10 -0
- data/lib/atomic_sidekiq/atomic_operation/lua_scripts/retrieve.lua +1 -1
- data/lib/atomic_sidekiq/atomic_operation/requeue.rb +1 -1
- data/lib/atomic_sidekiq/atomic_operation/retrieve.rb +6 -2
- data/lib/atomic_sidekiq/dead_job_collector.rb +13 -11
- data/lib/atomic_sidekiq/heartbeat.rb +34 -0
- data/lib/atomic_sidekiq/in_flight_keymaker.rb +30 -0
- data/lib/atomic_sidekiq/sidekiq/sidekiq.rb +6 -0
- data/lib/atomic_sidekiq/unit_of_work.rb +8 -4
- data/lib/atomic_sidekiq.rb +13 -10
- metadata +18 -12
- data/lib/atomic_sidekiq/sidekiq.rb +0 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3071f88191cdf61f92a11d1ffde7f5cab21db595
|
4
|
+
data.tar.gz: 850929d672c23bbfdd6ed645467ec2e49c4b4a99
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: cd7312265115acc03dd613e0623a74fdbc8ec3996de8177966af68300fad1a6d6e6a0aaf4e21d24a219ad0c189078fea097ce1d9d76550902fd87f0189076ef2
|
7
|
+
data.tar.gz: 5720a6e7b38226ee37cae38df9c94d49fe459d3890da868c3185415c822ff3dce8cb90dfbed4ade9d263b2f0ee418b868973995c85d9e6d232df638b3ad773ed
|
data/.gitignore
CHANGED
data/.rubocop.yml
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
AllCops:
|
2
|
+
TargetRubyVersion: 2.2
|
3
|
+
Exclude:
|
4
|
+
- bin/**/*
|
5
|
+
- spec/spec_helper.rb
|
6
|
+
- ./*.gemspec
|
7
|
+
- vendor/**/*
|
8
|
+
|
9
|
+
Style/StringLiterals:
|
10
|
+
EnforcedStyle: double_quotes
|
11
|
+
|
12
|
+
Naming/FileName:
|
13
|
+
Exclude:
|
14
|
+
- "lib/atomic-sidekiq.rb"
|
15
|
+
|
16
|
+
Style/DoubleNegation:
|
17
|
+
Enabled: false
|
18
|
+
|
19
|
+
Style/Documentation:
|
20
|
+
Enabled: false
|
21
|
+
|
22
|
+
Metrics/BlockLength:
|
23
|
+
Exclude:
|
24
|
+
- spec/**/*.rb
|
25
|
+
|
26
|
+
Metrics/LineLength:
|
27
|
+
Exclude:
|
28
|
+
- spec/**/*.rb
|
data/.travis.yml
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
language: ruby
|
2
|
+
sudo: false
|
3
|
+
cache: bundler
|
4
|
+
services:
|
5
|
+
- redis-server
|
6
|
+
before_install:
|
7
|
+
- gem install bundler
|
8
|
+
- gem update bundler
|
9
|
+
rvm:
|
10
|
+
- 2.2.2
|
11
|
+
- 2.3.5
|
12
|
+
- 2.4.2
|
13
|
+
- 2.5.0
|
14
|
+
script:
|
15
|
+
- bundle exec rubocop
|
16
|
+
- bundle exec rspec
|
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
@@ -8,10 +8,15 @@ GEM
|
|
8
8
|
remote: https://rubygems.org/
|
9
9
|
specs:
|
10
10
|
ast (2.4.0)
|
11
|
-
|
11
|
+
codecov (0.1.10)
|
12
|
+
json
|
13
|
+
simplecov
|
14
|
+
url
|
12
15
|
concurrent-ruby (1.0.5)
|
13
16
|
connection_pool (2.2.1)
|
14
17
|
diff-lcs (1.3)
|
18
|
+
docile (1.3.0)
|
19
|
+
json (2.1.0)
|
15
20
|
parallel (1.12.1)
|
16
21
|
parser (2.5.0.5)
|
17
22
|
ast (~> 2.4.0)
|
@@ -48,8 +53,14 @@ GEM
|
|
48
53
|
connection_pool (~> 2.2, >= 2.2.0)
|
49
54
|
rack-protection (>= 1.5.0)
|
50
55
|
redis (>= 3.3.5, < 5)
|
56
|
+
simplecov (0.16.1)
|
57
|
+
docile (~> 1.1)
|
58
|
+
json (>= 1.8, < 3)
|
59
|
+
simplecov-html (~> 0.10.0)
|
60
|
+
simplecov-html (0.10.2)
|
51
61
|
timecop (0.9.1)
|
52
62
|
unicode-display_width (1.3.0)
|
63
|
+
url (0.3.2)
|
53
64
|
|
54
65
|
PLATFORMS
|
55
66
|
ruby
|
@@ -57,11 +68,11 @@ PLATFORMS
|
|
57
68
|
DEPENDENCIES
|
58
69
|
atomic-sidekiq!
|
59
70
|
bundler (~> 1.12)
|
60
|
-
|
71
|
+
codecov
|
61
72
|
rake (~> 11.3)
|
62
|
-
rspec
|
63
|
-
rubocop
|
64
|
-
timecop
|
73
|
+
rspec
|
74
|
+
rubocop
|
75
|
+
timecop
|
65
76
|
|
66
77
|
BUNDLED WITH
|
67
78
|
1.16.1
|
data/README.md
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
[![Gem Version](https://badge.fury.io/rb/atomic-sidekiq.svg)](https://badge.fury.io/rb/atomic-sidekiq) [![Build Status](https://travis-ci.org/Colex/atomic-sidekiq.svg?branch=master)](https://travis-ci.org/Colex/atomic-sidekiq) [![codecov](https://codecov.io/gh/Colex/atomic-sidekiq/branch/master/graph/badge.svg)](https://codecov.io/gh/Colex/atomic-sidekiq)
|
2
|
+
|
1
3
|
# AtomicSidekiq
|
2
4
|
AtomicSidekiq implements a reliable way of processing jobs using Sidekiq. By default, Sidekiq will retrieve jobs from the queue by removing it from Redis. If the job fails to complete (e.g. the process terminates unexpectdly mid-job), the job will be lost forever. This can be acceptable in many applications, but some application require higher levels of reliability, hence AtomicSidekiq will not erase any job from Redis until it's acknowledged that they have finished - otherwise, they are re-scheduled.
|
3
5
|
|
@@ -29,6 +31,23 @@ Sidekiq.configure_server do |config|
|
|
29
31
|
end
|
30
32
|
```
|
31
33
|
|
34
|
+
## Heartbeat
|
35
|
+
For long running jobs that may run for an unpredictable amounts of time, you may send periodic heartbeats to reset the expiration time and allow the job to run for longer (if the job stops sending heartbeats and the expiration date run out, the job will be assumed lost and recovered). Example:
|
36
|
+
|
37
|
+
```ruby
|
38
|
+
class LongRunningWorker
|
39
|
+
include Sidekiq::Worker
|
40
|
+
include AtomicSidekiq::Heartbeat
|
41
|
+
|
42
|
+
def perform
|
43
|
+
(1..10_000).each do
|
44
|
+
ExampleClass.long_running_action!
|
45
|
+
heartbeat! # You can also give a specific timeout period, e.g. heartbeat!(1.hour)
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
```
|
50
|
+
|
32
51
|
## Benchmark
|
33
52
|
### Reliability
|
34
53
|
This benchmark tests Sidekiq's ability to recover from unexpected failures. The test script forces a failure randomly 1% of the time it's running a job and measures how many jobs are able to be completed:
|
data/atomic-sidekiq.gemspec
CHANGED
@@ -2,25 +2,25 @@
|
|
2
2
|
|
3
3
|
Gem::Specification.new do |s|
|
4
4
|
s.name = "atomic-sidekiq"
|
5
|
-
s.version = "1.
|
6
|
-
s.date = "2018-04-
|
5
|
+
s.version = "1.1.0"
|
6
|
+
s.date = "2018-04-02"
|
7
7
|
s.summary = "Reliable fetcher for Sidekiq"
|
8
8
|
s.description = "Reliable fetcher for Sidekiq"
|
9
9
|
s.homepage = "https://github.com/Colex/atomic-sidekiq"
|
10
10
|
s.authors = ["Alex Correia Santos"]
|
11
11
|
s.email = ["alex.santios@visiblealpha.com"]
|
12
|
-
s.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec
|
12
|
+
s.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec)/}) }
|
13
13
|
s.require_paths = ["lib"]
|
14
14
|
s.license = "MIT"
|
15
15
|
|
16
|
-
s.required_ruby_version = ">= 2.
|
16
|
+
s.required_ruby_version = ">= 2.2"
|
17
17
|
|
18
18
|
s.add_development_dependency "bundler", "~> 1.12"
|
19
19
|
s.add_development_dependency "rake", "~> 11.3"
|
20
20
|
s.add_development_dependency "rspec", "~> 3.6"
|
21
21
|
s.add_development_dependency "rubocop", "~> 0.54"
|
22
|
-
s.add_development_dependency "byebug", "~> 10.0"
|
23
22
|
s.add_development_dependency "timecop", "~> 0.9"
|
23
|
+
s.add_development_dependency "codecov", ">= 0.1.10"
|
24
24
|
|
25
25
|
s.add_runtime_dependency "sidekiq", "~> 5.0"
|
26
26
|
end
|
data/bin/sidekiqload
CHANGED
data/lib/atomic-sidekiq.rb
CHANGED
@@ -1 +1 @@
|
|
1
|
-
require_relative
|
1
|
+
require_relative "./atomic_sidekiq"
|
@@ -1,26 +1,30 @@
|
|
1
|
+
# rubocop:disable Style/ClassVars
|
1
2
|
module AtomicSidekiq
|
2
3
|
class AtomicFetch
|
3
|
-
IN_FLIGHT_KEY_PREFIX = "flight
|
4
|
-
DEFAULT_POLL_INTERVAL =
|
4
|
+
IN_FLIGHT_KEY_PREFIX = "flight".freeze
|
5
|
+
DEFAULT_POLL_INTERVAL = 5 # seconds
|
5
6
|
DEFAULT_EXPIRATION_TIME = 3600 # seconds
|
6
7
|
DEFAULT_COLLECTION_INTERVAL = 60 # seconds
|
7
8
|
|
8
|
-
def initialize(options)
|
9
|
-
@
|
10
|
-
|
9
|
+
def initialize(options, in_flight_keymaker: nil)
|
10
|
+
@keymaker = in_flight_keymaker ||
|
11
|
+
InFlightKeymaker.new(IN_FLIGHT_KEY_PREFIX)
|
12
|
+
|
13
|
+
@retrieve_op = AtomicOperation::Retrieve.new(
|
14
|
+
in_flight_keymaker: keymaker
|
15
|
+
)
|
11
16
|
|
12
|
-
|
13
|
-
@
|
14
|
-
@collection_interval = atomic_fetch_opts.fetch(:collection_wait_time, DEFAULT_COLLECTION_INTERVAL)
|
15
|
-
@poll_interval = atomic_fetch_opts.fetch(:poll_interval, DEFAULT_POLL_INTERVAL)
|
17
|
+
@queues ||= options[:queues].map { |q| "queue:#{q}" }
|
18
|
+
@strictly_ordered_queues = !!options[:strict]
|
16
19
|
@@next_collection ||= Time.now
|
17
|
-
|
20
|
+
|
21
|
+
configure_atomic_fetch(options.fetch(:atomic_fetch, {}))
|
18
22
|
end
|
19
23
|
|
20
24
|
def retrieve_work
|
21
25
|
collect_dead_jobs!
|
22
26
|
work = retrieve_op.perform(ordered_queues, expire_at)
|
23
|
-
return UnitOfWork.new(*work) if work
|
27
|
+
return UnitOfWork.new(*work, in_flight_keymaker: keymaker) if work
|
24
28
|
sleep(poll_interval)
|
25
29
|
nil
|
26
30
|
end
|
@@ -28,10 +32,14 @@ module AtomicSidekiq
|
|
28
32
|
private
|
29
33
|
|
30
34
|
attr_reader :retrieve_op, :queues, :strictly_ordered_queues,
|
31
|
-
|
35
|
+
:collection_interval, :poll_interval, :expiration_time,
|
36
|
+
:keymaker
|
32
37
|
|
33
|
-
def
|
34
|
-
@
|
38
|
+
def configure_atomic_fetch(options)
|
39
|
+
@expiration_time = options[:expiration_time] || DEFAULT_EXPIRATION_TIME
|
40
|
+
@collection_interval = options[:collection_wait_time] ||
|
41
|
+
DEFAULT_COLLECTION_INTERVAL
|
42
|
+
@poll_interval = options[:poll_interval] || DEFAULT_POLL_INTERVAL
|
35
43
|
end
|
36
44
|
|
37
45
|
def ordered_queues
|
@@ -45,7 +53,7 @@ module AtomicSidekiq
|
|
45
53
|
def collect_dead_jobs!
|
46
54
|
return if @@next_collection > Time.now
|
47
55
|
@@next_collection = Time.now + collection_interval
|
48
|
-
DeadJobCollector.collect!(ordered_queues)
|
56
|
+
DeadJobCollector.collect!(ordered_queues, in_flight_keymaker: keymaker)
|
49
57
|
end
|
50
58
|
|
51
59
|
def expire_at
|
@@ -53,3 +61,4 @@ module AtomicSidekiq
|
|
53
61
|
end
|
54
62
|
end
|
55
63
|
end
|
64
|
+
# rubocop:enable Style/ClassVars
|
@@ -1,21 +1,16 @@
|
|
1
1
|
module AtomicSidekiq
|
2
2
|
module AtomicOperation
|
3
3
|
class Base
|
4
|
-
def initialize(
|
5
|
-
@
|
4
|
+
def initialize(in_flight_keymaker:)
|
5
|
+
@in_flight_keymaker = in_flight_keymaker
|
6
6
|
end
|
7
7
|
|
8
8
|
protected
|
9
9
|
|
10
|
-
attr_reader :
|
10
|
+
attr_reader :in_flight_keymaker
|
11
11
|
|
12
|
-
def redis
|
13
|
-
Sidekiq.redis { |conn|
|
14
|
-
end
|
15
|
-
|
16
|
-
def in_flight_job_key(queue, job)
|
17
|
-
jid = JSON.parse(job)['jid']
|
18
|
-
"#{in_flight_prefix}#{queue}:#{jid}"
|
12
|
+
def redis
|
13
|
+
Sidekiq.redis { |conn| yield(conn) }
|
19
14
|
end
|
20
15
|
end
|
21
16
|
end
|
@@ -1,8 +1,13 @@
|
|
1
1
|
module AtomicSidekiq
|
2
2
|
module AtomicOperation
|
3
3
|
class Expire < Base
|
4
|
+
EXPIRE_SCRIPT = File.read(
|
5
|
+
File.join(File.dirname(__FILE__),
|
6
|
+
"./lua_scripts/expire.lua")
|
7
|
+
)
|
8
|
+
|
4
9
|
def initialize
|
5
|
-
super(
|
10
|
+
super(in_flight_keymaker: nil)
|
6
11
|
end
|
7
12
|
|
8
13
|
def perform(queue, in_flight_key)
|
@@ -10,10 +15,6 @@ module AtomicSidekiq
|
|
10
15
|
conn.eval(EXPIRE_SCRIPT, [queue, in_flight_key], [Time.now.utc.to_i])
|
11
16
|
end
|
12
17
|
end
|
13
|
-
|
14
|
-
private
|
15
|
-
|
16
|
-
EXPIRE_SCRIPT = File.read(File.join(File.dirname(__FILE__), './lua_scripts/expire.lua'))
|
17
18
|
end
|
18
19
|
end
|
19
20
|
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
module AtomicSidekiq
|
2
|
+
module AtomicOperation
|
3
|
+
class Heartbeat < Base
|
4
|
+
HEARTBEAT_SCRIPT = File.read(
|
5
|
+
File.join(File.dirname(__FILE__),
|
6
|
+
"./lua_scripts/heartbeat.lua")
|
7
|
+
)
|
8
|
+
|
9
|
+
def perform(jid:, timeout:)
|
10
|
+
key = in_flight_job_key(jid)
|
11
|
+
return unless key
|
12
|
+
redis do |conn|
|
13
|
+
conn.eval(HEARTBEAT_SCRIPT, [key], [expiration_date(timeout)])
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
private
|
18
|
+
|
19
|
+
def expiration_date(timeout)
|
20
|
+
Time.now.utc.to_i + timeout
|
21
|
+
end
|
22
|
+
|
23
|
+
def in_flight_job_key(jid)
|
24
|
+
matcher = in_flight_keymaker.job_matcher(jid)
|
25
|
+
it = 0
|
26
|
+
loop do
|
27
|
+
it, keys = redis { |conn| conn.scan(it, match: matcher) }
|
28
|
+
return keys[0] if keys.count > 0
|
29
|
+
it = it.to_i
|
30
|
+
return if it.zero?
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,10 @@
|
|
1
|
+
local in_flight_key = KEYS[1]
|
2
|
+
local expire_at = ARGV[1]
|
3
|
+
|
4
|
+
local job = redis.call('get', in_flight_key)
|
5
|
+
if (not job) then return nil end
|
6
|
+
|
7
|
+
job = string.gsub(job, '("expire_at":)[0-9]*', '%1'..expire_at)
|
8
|
+
redis.call('set', in_flight_key, job)
|
9
|
+
|
10
|
+
return job
|
@@ -6,6 +6,6 @@ local job = redis.call('lpop', queue)
|
|
6
6
|
if (not job) then return nil end
|
7
7
|
job = job:sub(1,-2)..',"expire_at":'..expire_at.."}"
|
8
8
|
|
9
|
-
local flight_key = flight..
|
9
|
+
local flight_key = flight..string.match(job, '"jid":"([^"]*)"')
|
10
10
|
redis.call('set', flight_key, job)
|
11
11
|
return { queue, job }
|
@@ -1,6 +1,11 @@
|
|
1
1
|
module AtomicSidekiq
|
2
2
|
module AtomicOperation
|
3
3
|
class Retrieve < Base
|
4
|
+
RETRIEVE_SCRIPT = File.read(
|
5
|
+
File.join(File.dirname(__FILE__),
|
6
|
+
"./lua_scripts/retrieve.lua")
|
7
|
+
)
|
8
|
+
|
4
9
|
def perform(queues, expire_at)
|
5
10
|
queues.each do |queue|
|
6
11
|
res = retrieve_from_queue(queue, expire_at.to_i)
|
@@ -11,9 +16,8 @@ module AtomicSidekiq
|
|
11
16
|
|
12
17
|
private
|
13
18
|
|
14
|
-
RETRIEVE_SCRIPT = File.read(File.join(File.dirname(__FILE__), './lua_scripts/retrieve.lua'))
|
15
|
-
|
16
19
|
def retrieve_from_queue(queue, expire_at)
|
20
|
+
in_flight_prefix = in_flight_keymaker.queue_prefix(queue)
|
17
21
|
redis do |conn|
|
18
22
|
conn.eval(RETRIEVE_SCRIPT, [queue, in_flight_prefix], [expire_at])
|
19
23
|
end
|
@@ -1,15 +1,17 @@
|
|
1
1
|
module AtomicSidekiq
|
2
2
|
class DeadJobCollector
|
3
3
|
class << self
|
4
|
-
def collect!(queues)
|
5
|
-
queues.each
|
4
|
+
def collect!(queues, in_flight_keymaker:)
|
5
|
+
queues.each do |q|
|
6
|
+
new(q, in_flight_keymaker: in_flight_keymaker).collect!
|
7
|
+
end
|
6
8
|
end
|
7
9
|
end
|
8
10
|
|
9
|
-
def initialize(queue,
|
10
|
-
@queue
|
11
|
-
@
|
12
|
-
@expire_op
|
11
|
+
def initialize(queue, in_flight_keymaker:)
|
12
|
+
@queue = queue
|
13
|
+
@in_flight_keymaker = in_flight_keymaker
|
14
|
+
@expire_op = AtomicOperation::Expire.new
|
13
15
|
end
|
14
16
|
|
15
17
|
def collect!
|
@@ -18,26 +20,26 @@ module AtomicSidekiq
|
|
18
20
|
|
19
21
|
private
|
20
22
|
|
21
|
-
attr_reader :queue, :
|
23
|
+
attr_reader :queue, :in_flight_keymaker, :expire_op
|
22
24
|
|
23
25
|
def expire!(job_key)
|
24
26
|
expire_op.perform(queue, job_key)
|
25
27
|
end
|
26
28
|
|
27
|
-
def each_keys
|
29
|
+
def each_keys
|
28
30
|
it = 0
|
29
31
|
Sidekiq.redis do |conn|
|
30
32
|
loop do
|
31
33
|
it, job_keys = conn.scan(it, match: keys_prefix)
|
32
34
|
it = it.to_i
|
33
|
-
job_keys.each { |job_key|
|
34
|
-
break if it
|
35
|
+
job_keys.each { |job_key| yield(job_key) }
|
36
|
+
break if it.zero?
|
35
37
|
end
|
36
38
|
end
|
37
39
|
end
|
38
40
|
|
39
41
|
def keys_prefix
|
40
|
-
|
42
|
+
in_flight_keymaker.queue_matcher(queue)
|
41
43
|
end
|
42
44
|
end
|
43
45
|
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
module AtomicSidekiq
|
2
|
+
module Heartbeat
|
3
|
+
def self.included(base)
|
4
|
+
base.send :include, InstanceMethods
|
5
|
+
end
|
6
|
+
|
7
|
+
module InstanceMethods
|
8
|
+
def heartbeat!(timeout = nil)
|
9
|
+
heartbeat_operation.perform(
|
10
|
+
jid: jid,
|
11
|
+
timeout: timeout || default_heartbeat_timeout
|
12
|
+
)
|
13
|
+
end
|
14
|
+
|
15
|
+
private
|
16
|
+
|
17
|
+
def default_heartbeat_timeout
|
18
|
+
AtomicSidekiq::AtomicFetch::DEFAULT_EXPIRATION_TIME
|
19
|
+
end
|
20
|
+
|
21
|
+
def heartbeat_operation
|
22
|
+
@heartbeat_operation ||= AtomicSidekiq::AtomicOperation::Heartbeat.new(
|
23
|
+
in_flight_keymaker: keymaker
|
24
|
+
)
|
25
|
+
end
|
26
|
+
|
27
|
+
def keymaker
|
28
|
+
@keymaker ||= AtomicSidekiq::InFlightKeymaker.new(
|
29
|
+
AtomicSidekiq::AtomicFetch::IN_FLIGHT_KEY_PREFIX
|
30
|
+
)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
module AtomicSidekiq
|
2
|
+
class InFlightKeymaker
|
3
|
+
def initialize(key_prefix)
|
4
|
+
@key_prefix = key_prefix
|
5
|
+
end
|
6
|
+
|
7
|
+
def queue_prefix(queue)
|
8
|
+
normalized_name = queue.gsub(/queue:/, "")
|
9
|
+
"#{key_prefix}:#{normalized_name}:"
|
10
|
+
end
|
11
|
+
|
12
|
+
def queue_matcher(queue)
|
13
|
+
"#{queue_prefix(queue)}*"
|
14
|
+
end
|
15
|
+
|
16
|
+
def job_key(job)
|
17
|
+
obj = job
|
18
|
+
obj = JSON.parse(obj) if job.is_a?(String)
|
19
|
+
"#{key_prefix}:#{obj['queue']}:#{obj['jid']}"
|
20
|
+
end
|
21
|
+
|
22
|
+
def job_matcher(jid)
|
23
|
+
"#{key_prefix}:*:#{jid}"
|
24
|
+
end
|
25
|
+
|
26
|
+
private
|
27
|
+
|
28
|
+
attr_reader :key_prefix
|
29
|
+
end
|
30
|
+
end
|
@@ -2,15 +2,19 @@ module AtomicSidekiq
|
|
2
2
|
class UnitOfWork
|
3
3
|
attr_reader :queue, :job
|
4
4
|
|
5
|
-
def initialize(queue = nil, job = nil,
|
5
|
+
def initialize(queue = nil, job = nil, in_flight_keymaker:)
|
6
6
|
@queue = queue
|
7
7
|
@job = job
|
8
|
-
@acknowledge_op = AtomicOperation::Acknowledge.new(
|
9
|
-
|
8
|
+
@acknowledge_op = AtomicOperation::Acknowledge.new(
|
9
|
+
in_flight_keymaker: in_flight_keymaker
|
10
|
+
)
|
11
|
+
@requeue_op = AtomicOperation::Requeue.new(
|
12
|
+
in_flight_keymaker: in_flight_keymaker
|
13
|
+
)
|
10
14
|
end
|
11
15
|
|
12
16
|
def acknowledge
|
13
|
-
acknowledge_op.perform(
|
17
|
+
acknowledge_op.perform(job: job)
|
14
18
|
end
|
15
19
|
|
16
20
|
def queue_name
|
data/lib/atomic_sidekiq.rb
CHANGED
@@ -1,13 +1,16 @@
|
|
1
|
-
require
|
2
|
-
require_relative
|
3
|
-
require_relative
|
4
|
-
require_relative
|
5
|
-
require_relative
|
6
|
-
require_relative
|
7
|
-
require_relative
|
8
|
-
require_relative
|
9
|
-
require_relative
|
10
|
-
require_relative
|
1
|
+
require "sidekiq"
|
2
|
+
require_relative "atomic_sidekiq/sidekiq/sidekiq"
|
3
|
+
require_relative "atomic_sidekiq/in_flight_keymaker"
|
4
|
+
require_relative "atomic_sidekiq/unit_of_work"
|
5
|
+
require_relative "atomic_sidekiq/atomic_fetch"
|
6
|
+
require_relative "atomic_sidekiq/dead_job_collector"
|
7
|
+
require_relative "atomic_sidekiq/heartbeat"
|
8
|
+
require_relative "atomic_sidekiq/atomic_operation/base"
|
9
|
+
require_relative "atomic_sidekiq/atomic_operation/acknowledge"
|
10
|
+
require_relative "atomic_sidekiq/atomic_operation/requeue"
|
11
|
+
require_relative "atomic_sidekiq/atomic_operation/retrieve"
|
12
|
+
require_relative "atomic_sidekiq/atomic_operation/expire"
|
13
|
+
require_relative "atomic_sidekiq/atomic_operation/heartbeat"
|
11
14
|
|
12
15
|
module AtomicSidekiq
|
13
16
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: atomic-sidekiq
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Alex Correia Santos
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-04-
|
11
|
+
date: 2018-04-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -67,33 +67,33 @@ dependencies:
|
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '0.54'
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
|
-
name:
|
70
|
+
name: timecop
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
73
|
- - "~>"
|
74
74
|
- !ruby/object:Gem::Version
|
75
|
-
version: '
|
75
|
+
version: '0.9'
|
76
76
|
type: :development
|
77
77
|
prerelease: false
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
80
80
|
- - "~>"
|
81
81
|
- !ruby/object:Gem::Version
|
82
|
-
version: '
|
82
|
+
version: '0.9'
|
83
83
|
- !ruby/object:Gem::Dependency
|
84
|
-
name:
|
84
|
+
name: codecov
|
85
85
|
requirement: !ruby/object:Gem::Requirement
|
86
86
|
requirements:
|
87
|
-
- - "
|
87
|
+
- - ">="
|
88
88
|
- !ruby/object:Gem::Version
|
89
|
-
version:
|
89
|
+
version: 0.1.10
|
90
90
|
type: :development
|
91
91
|
prerelease: false
|
92
92
|
version_requirements: !ruby/object:Gem::Requirement
|
93
93
|
requirements:
|
94
|
-
- - "
|
94
|
+
- - ">="
|
95
95
|
- !ruby/object:Gem::Version
|
96
|
-
version:
|
96
|
+
version: 0.1.10
|
97
97
|
- !ruby/object:Gem::Dependency
|
98
98
|
name: sidekiq
|
99
99
|
requirement: !ruby/object:Gem::Requirement
|
@@ -117,6 +117,8 @@ extra_rdoc_files: []
|
|
117
117
|
files:
|
118
118
|
- ".gitignore"
|
119
119
|
- ".rspec"
|
120
|
+
- ".rubocop.yml"
|
121
|
+
- ".travis.yml"
|
120
122
|
- Dockerfile
|
121
123
|
- Gemfile
|
122
124
|
- Gemfile.lock
|
@@ -135,12 +137,16 @@ files:
|
|
135
137
|
- lib/atomic_sidekiq/atomic_operation/acknowledge.rb
|
136
138
|
- lib/atomic_sidekiq/atomic_operation/base.rb
|
137
139
|
- lib/atomic_sidekiq/atomic_operation/expire.rb
|
140
|
+
- lib/atomic_sidekiq/atomic_operation/heartbeat.rb
|
138
141
|
- lib/atomic_sidekiq/atomic_operation/lua_scripts/expire.lua
|
142
|
+
- lib/atomic_sidekiq/atomic_operation/lua_scripts/heartbeat.lua
|
139
143
|
- lib/atomic_sidekiq/atomic_operation/lua_scripts/retrieve.lua
|
140
144
|
- lib/atomic_sidekiq/atomic_operation/requeue.rb
|
141
145
|
- lib/atomic_sidekiq/atomic_operation/retrieve.rb
|
142
146
|
- lib/atomic_sidekiq/dead_job_collector.rb
|
143
|
-
- lib/atomic_sidekiq/
|
147
|
+
- lib/atomic_sidekiq/heartbeat.rb
|
148
|
+
- lib/atomic_sidekiq/in_flight_keymaker.rb
|
149
|
+
- lib/atomic_sidekiq/sidekiq/sidekiq.rb
|
144
150
|
- lib/atomic_sidekiq/unit_of_work.rb
|
145
151
|
homepage: https://github.com/Colex/atomic-sidekiq
|
146
152
|
licenses:
|
@@ -154,7 +160,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
154
160
|
requirements:
|
155
161
|
- - ">="
|
156
162
|
- !ruby/object:Gem::Version
|
157
|
-
version: '2.
|
163
|
+
version: '2.2'
|
158
164
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
159
165
|
requirements:
|
160
166
|
- - ">="
|