multi-background-job 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.github/workflows/specs.yml +35 -0
- data/.gitignore +10 -0
- data/.rspec +1 -0
- data/.tool-versions +1 -0
- data/Gemfile +12 -0
- data/Gemfile.lock +55 -0
- data/LICENSE.txt +21 -0
- data/README.md +137 -0
- data/Rakefile +2 -0
- data/bin/console +13 -0
- data/bin/setup +6 -0
- data/lib/multi-background-job.rb +3 -0
- data/lib/multi_background_job.rb +89 -0
- data/lib/multi_background_job/adapters/adapter.rb +23 -0
- data/lib/multi_background_job/adapters/faktory.rb +111 -0
- data/lib/multi_background_job/adapters/sidekiq.rb +91 -0
- data/lib/multi_background_job/config.rb +152 -0
- data/lib/multi_background_job/errors.rb +25 -0
- data/lib/multi_background_job/lock.rb +137 -0
- data/lib/multi_background_job/lock_digest.rb +39 -0
- data/lib/multi_background_job/middleware/unique_job.rb +69 -0
- data/lib/multi_background_job/middleware/unique_job/faktory.rb +41 -0
- data/lib/multi_background_job/middleware/unique_job/sidekiq.rb +48 -0
- data/lib/multi_background_job/middleware_chain.rb +109 -0
- data/lib/multi_background_job/unique_job.rb +84 -0
- data/lib/multi_background_job/version.rb +5 -0
- data/lib/multi_background_job/worker.rb +114 -0
- data/lib/multi_background_job/workers/faktory.rb +28 -0
- data/lib/multi_background_job/workers/shared_class_methods.rb +26 -0
- data/lib/multi_background_job/workers/sidekiq.rb +28 -0
- data/multi-background-job.gemspec +39 -0
- metadata +122 -0
@@ -0,0 +1,41 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# Server midleware for Faktory
|
4
|
+
#
|
5
|
+
# @see https://github.com/contribsys/faktory_worker_ruby/wiki/Middleware
|
6
|
+
module MultiBackgroundJob
|
7
|
+
module Middleware
|
8
|
+
class UniqueJob
|
9
|
+
module Faktory
|
10
|
+
def self.bootstrap
|
11
|
+
if defined?(::Faktory)
|
12
|
+
::Faktory.configure_worker do |config|
|
13
|
+
config.worker_middleware do |chain|
|
14
|
+
chain.add Worker
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
# Worker middleware runs around the execution of a job
|
21
|
+
class Worker
|
22
|
+
def call(_jobinst, payload)
|
23
|
+
if payload.is_a?(Hash) && (unique_lock = unique_job_lock(payload))
|
24
|
+
unique_lock.unlock
|
25
|
+
end
|
26
|
+
yield
|
27
|
+
end
|
28
|
+
|
29
|
+
protected
|
30
|
+
|
31
|
+
def unique_job_lock(payload)
|
32
|
+
return unless payload['uniq'].is_a?(Hash)
|
33
|
+
|
34
|
+
unique_job = ::MultiBackgroundJob::UniqueJob.coerce(payload['uniq'])
|
35
|
+
unique_job&.lock
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
|
4
|
+
# Provides the Sidekiq middleware that make the unique job control work
|
5
|
+
#
|
6
|
+
# @see https://github.com/contribsys/faktory_worker_ruby/wiki/Middleware
|
7
|
+
module MultiBackgroundJob
|
8
|
+
module Middleware
|
9
|
+
class UniqueJob
|
10
|
+
module Sidekiq
|
11
|
+
def self.bootstrap
|
12
|
+
if defined?(::Sidekiq)
|
13
|
+
::Sidekiq.configure_worker do |config|
|
14
|
+
config.worker_middleware do |chain|
|
15
|
+
chain.add Worker
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
# Worker middleware runs around the execution of a job
|
22
|
+
class Worker
|
23
|
+
# @param worker [Object] the worker instance
|
24
|
+
# @param job [Hash] the full job payload
|
25
|
+
# * @see https://github.com/mperham/sidekiq/wiki/Job-Format
|
26
|
+
# @param queue [String] the name of the queue the job was pulled from
|
27
|
+
# @yield the next middleware in the chain or worker `perform` method
|
28
|
+
# @return [Void]
|
29
|
+
def call(_worker, job, _queue)
|
30
|
+
if job.is_a?(Hash) && (unique_lock = unique_job_lock(job))
|
31
|
+
unique_lock.unlock
|
32
|
+
end
|
33
|
+
yield
|
34
|
+
end
|
35
|
+
|
36
|
+
protected
|
37
|
+
|
38
|
+
def unique_job_lock(job)
|
39
|
+
return unless job['uniq'].is_a?(Hash)
|
40
|
+
|
41
|
+
unique_job = ::MultiBackgroundJob::UniqueJob.coerce(job['uniq'])
|
42
|
+
unique_job&.lock
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
@@ -0,0 +1,109 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module MultiBackgroundJob
|
4
|
+
# Middleware is code configured to run before/after push a new job.
|
5
|
+
# It is patterned after Rack middleware for some modification before push the job to the server
|
6
|
+
#
|
7
|
+
# To add a middleware:
|
8
|
+
#
|
9
|
+
# MultiBackgroundJob.configure do |config|
|
10
|
+
# config.middleware do |chain|
|
11
|
+
# chain.add MyMiddleware
|
12
|
+
# end
|
13
|
+
# end
|
14
|
+
#
|
15
|
+
# This is an example of a minimal middleware, note the method must return the result
|
16
|
+
# or the job will not push the server.
|
17
|
+
#
|
18
|
+
# class MyMiddleware
|
19
|
+
# def call(job, conn_pool)
|
20
|
+
# puts "Before push"
|
21
|
+
# result = yield
|
22
|
+
# puts "After push"
|
23
|
+
# result
|
24
|
+
# end
|
25
|
+
# end
|
26
|
+
#
|
27
|
+
class MiddlewareChain
|
28
|
+
include Enumerable
|
29
|
+
attr_reader :entries
|
30
|
+
|
31
|
+
def initialize_copy(copy)
|
32
|
+
copy.instance_variable_set(:@entries, entries.dup)
|
33
|
+
end
|
34
|
+
|
35
|
+
def each(&block)
|
36
|
+
entries.each(&block)
|
37
|
+
end
|
38
|
+
|
39
|
+
def initialize
|
40
|
+
@entries = []
|
41
|
+
yield self if block_given?
|
42
|
+
end
|
43
|
+
|
44
|
+
def remove(klass)
|
45
|
+
entries.delete_if { |entry| entry.klass == klass }
|
46
|
+
end
|
47
|
+
|
48
|
+
def add(klass, *args)
|
49
|
+
remove(klass) if exists?(klass)
|
50
|
+
entries << Entry.new(klass, *args)
|
51
|
+
end
|
52
|
+
|
53
|
+
def prepend(klass, *args)
|
54
|
+
remove(klass) if exists?(klass)
|
55
|
+
entries.insert(0, Entry.new(klass, *args))
|
56
|
+
end
|
57
|
+
|
58
|
+
def insert_before(oldklass, newklass, *args)
|
59
|
+
i = entries.index { |entry| entry.klass == newklass }
|
60
|
+
new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
|
61
|
+
i = entries.index { |entry| entry.klass == oldklass } || 0
|
62
|
+
entries.insert(i, new_entry)
|
63
|
+
end
|
64
|
+
|
65
|
+
def insert_after(oldklass, newklass, *args)
|
66
|
+
i = entries.index { |entry| entry.klass == newklass }
|
67
|
+
new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
|
68
|
+
i = entries.index { |entry| entry.klass == oldklass } || entries.count - 1
|
69
|
+
entries.insert(i+1, new_entry)
|
70
|
+
end
|
71
|
+
|
72
|
+
def exists?(klass)
|
73
|
+
any? { |entry| entry.klass == klass }
|
74
|
+
end
|
75
|
+
|
76
|
+
def retrieve
|
77
|
+
map(&:make_new)
|
78
|
+
end
|
79
|
+
|
80
|
+
def clear
|
81
|
+
entries.clear
|
82
|
+
end
|
83
|
+
|
84
|
+
def invoke(*args)
|
85
|
+
chain = retrieve.dup
|
86
|
+
traverse_chain = lambda do
|
87
|
+
if chain.empty?
|
88
|
+
yield
|
89
|
+
else
|
90
|
+
chain.pop.call(*args, &traverse_chain)
|
91
|
+
end
|
92
|
+
end
|
93
|
+
traverse_chain.call
|
94
|
+
end
|
95
|
+
|
96
|
+
class Entry
|
97
|
+
attr_reader :klass, :args
|
98
|
+
|
99
|
+
def initialize(klass, *args)
|
100
|
+
@klass = klass
|
101
|
+
@args = args
|
102
|
+
end
|
103
|
+
|
104
|
+
def make_new
|
105
|
+
@klass.new(*@args)
|
106
|
+
end
|
107
|
+
end
|
108
|
+
end
|
109
|
+
end
|
@@ -0,0 +1,84 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative './lock'
|
4
|
+
|
5
|
+
module MultiBackgroundJob
|
6
|
+
class UniqueJob
|
7
|
+
VALID_OPTIONS = {
|
8
|
+
across: %i[queue systemwide],
|
9
|
+
timeout: 604800, # 1 week
|
10
|
+
unlock_policy: %i[success start],
|
11
|
+
}.freeze
|
12
|
+
|
13
|
+
attr_reader :across, :timeout, :unlock_policy, :lock
|
14
|
+
|
15
|
+
# @options [Hash] Unique definitions
|
16
|
+
# @option [Symbol] :across Valid options are :queue and :systemwide. If jobs should not to be duplicated on
|
17
|
+
# current queue or the entire system
|
18
|
+
# @option [Integer] :timeout Amount of times in seconds. Timeout decides how long to wait for acquiring the lock.
|
19
|
+
# A default timeout is defined to 1 week so unique locks won't last forever.
|
20
|
+
# @option [Symbol] :unlock_policy Control when the unique lock is removed. The default value is `success`.
|
21
|
+
# The job will not unlock until it executes successfully, it will remain locked even if it raises an error and
|
22
|
+
# goes into the retry queue. The alternative value is `start` the job will unlock right before it starts executing
|
23
|
+
def initialize(across: :queue, timeout: nil, unlock_policy: :success, lock: nil)
|
24
|
+
unless VALID_OPTIONS[:across].include?(across.to_sym)
|
25
|
+
raise Error, format('Invalid `across: %<given>p` option. Only %<expected>p are allowed.',
|
26
|
+
given: across,
|
27
|
+
expected: VALID_OPTIONS[:across])
|
28
|
+
end
|
29
|
+
unless VALID_OPTIONS[:unlock_policy].include?(unlock_policy.to_sym)
|
30
|
+
raise Error, format('Invalid `unlock_policy: %<given>p` option. Only %<expected>p are allowed.',
|
31
|
+
given: unlock_policy,
|
32
|
+
expected: VALID_OPTIONS[:unlock_policy])
|
33
|
+
end
|
34
|
+
timeout = VALID_OPTIONS[:timeout] if timeout.to_i <= 0
|
35
|
+
|
36
|
+
@across = across.to_sym
|
37
|
+
@timeout = timeout.to_i
|
38
|
+
@unlock_policy = unlock_policy.to_sym
|
39
|
+
@lock = lock if lock.is_a?(MultiBackgroundJob::Lock)
|
40
|
+
end
|
41
|
+
|
42
|
+
def self.coerce(value)
|
43
|
+
return unless value.is_a?(Hash)
|
44
|
+
|
45
|
+
new(
|
46
|
+
across: (value['across'] || value[:across] || :queue).to_sym,
|
47
|
+
timeout: (value['timeout'] || value[:timeout] || nil),
|
48
|
+
unlock_policy: (value['unlock_policy'] || value[:unlock_policy] || :success).to_sym,
|
49
|
+
).tap do |instance|
|
50
|
+
instance.lock = value['lock'] || value[:lock]
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
def ttl
|
55
|
+
Time.now.to_f + timeout
|
56
|
+
end
|
57
|
+
|
58
|
+
def to_hash
|
59
|
+
{
|
60
|
+
'across' => (across.to_s if across),
|
61
|
+
'timeout' => timeout,
|
62
|
+
'unlock_policy' => (unlock_policy.to_s if unlock_policy),
|
63
|
+
}.tap do |hash|
|
64
|
+
hash['lock'] = lock.to_hash if lock
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
def eql?(other)
|
69
|
+
return false unless other.is_a?(self.class)
|
70
|
+
|
71
|
+
[across, timeout, unlock_policy] == [other.across, other.timeout, other.unlock_policy]
|
72
|
+
end
|
73
|
+
alias == eql?
|
74
|
+
|
75
|
+
def lock=(value)
|
76
|
+
@lock = case value
|
77
|
+
when MultiBackgroundJob::Lock then value
|
78
|
+
when Hash then MultiBackgroundJob::Lock.coerce(value)
|
79
|
+
else
|
80
|
+
nil
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
@@ -0,0 +1,114 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative './unique_job'
|
4
|
+
|
5
|
+
module MultiBackgroundJob
|
6
|
+
class Worker
|
7
|
+
attr_reader :options, :payload, :worker_class, :unique_job
|
8
|
+
|
9
|
+
attr_reader :arguments
|
10
|
+
|
11
|
+
def initialize(worker_class, **options)
|
12
|
+
@worker_class = worker_class
|
13
|
+
@options = options
|
14
|
+
@payload = {}
|
15
|
+
unique(@options.delete(:uniq)) if @options.key?(:uniq)
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.coerce(service:, payload:, **opts)
|
19
|
+
SERVICES.fetch(service).coerce_to_worker(payload, **opts)
|
20
|
+
end
|
21
|
+
|
22
|
+
%i[created_at enqueued_at].each do |method_name|
|
23
|
+
define_method method_name do |value|
|
24
|
+
@payload[method_name.to_s] = \
|
25
|
+
case value
|
26
|
+
when Numeric then value.to_f
|
27
|
+
when String then Time.parse(value).to_f
|
28
|
+
when Time, DateTime then value.to_f
|
29
|
+
else
|
30
|
+
raise ArgumentError, format('The %<v>p is not a valid value for %<m>s.', v: value, m: method_name)
|
31
|
+
end
|
32
|
+
|
33
|
+
self
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
# Adds arguments to the job
|
38
|
+
# @return self
|
39
|
+
def with_args(*args)
|
40
|
+
@payload['args'] = args
|
41
|
+
|
42
|
+
self
|
43
|
+
end
|
44
|
+
|
45
|
+
# Schedule the time when a job will be executed. Jobs which are scheduled in the past are enqueued for immediate execution.
|
46
|
+
# @param timestamp [Numeric] timestamp, numeric or something that acts numeric.
|
47
|
+
# @return self
|
48
|
+
def in(timestamp)
|
49
|
+
now = Time.now.to_f
|
50
|
+
timestamp = Time.parse(timestamp) if timestamp.is_a?(String)
|
51
|
+
int = timestamp.respond_to?(:strftime) ? timestamp.to_f : now + timestamp.to_f
|
52
|
+
return self if int <= now
|
53
|
+
|
54
|
+
@payload['at'] = int
|
55
|
+
@payload['created_at'] = now
|
56
|
+
|
57
|
+
self
|
58
|
+
end
|
59
|
+
alias_method :at, :in
|
60
|
+
|
61
|
+
# Wrap uniq options
|
62
|
+
#
|
63
|
+
# @param value [Hash] Unique configurations with `across`, `timeout` and `unlock_policy`
|
64
|
+
# @return self
|
65
|
+
def unique(value)
|
66
|
+
value = {} if value == true
|
67
|
+
@unique_job = \
|
68
|
+
case value
|
69
|
+
when Hash then UniqueJob.coerce(value)
|
70
|
+
when UniqueJob then value
|
71
|
+
else
|
72
|
+
nil
|
73
|
+
end
|
74
|
+
|
75
|
+
self
|
76
|
+
end
|
77
|
+
|
78
|
+
def with_job_jid(jid = nil)
|
79
|
+
@payload['jid'] ||= jid || MultiBackgroundJob.jid
|
80
|
+
|
81
|
+
self
|
82
|
+
end
|
83
|
+
|
84
|
+
# @param :to [Symbol] Adapter key
|
85
|
+
# @return Response of service
|
86
|
+
# @see MultiBackgroundJob::Adapters::** for more details
|
87
|
+
def push(to: nil)
|
88
|
+
to ||= options[:service]
|
89
|
+
unless SERVICES.include?(to)
|
90
|
+
raise Error, format('Service %<to>p is not implemented. Please use one of %<list>p', to: to, list: SERVICES.keys)
|
91
|
+
end
|
92
|
+
|
93
|
+
@payload['created_at'] ||= Time.now.to_f
|
94
|
+
worker_to_push = with_job_jid
|
95
|
+
MultiBackgroundJob.config.middleware.invoke(worker_to_push, to) do
|
96
|
+
SERVICES[to].push(worker_to_push)
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
def eql?(other)
|
101
|
+
return false unless other.is_a?(self.class)
|
102
|
+
|
103
|
+
worker_class == other.worker_class && \
|
104
|
+
payload == other.payload &&
|
105
|
+
options == other.options &&
|
106
|
+
unique_job == other.unique_job
|
107
|
+
end
|
108
|
+
alias == eql?
|
109
|
+
|
110
|
+
def unique_job?
|
111
|
+
unique_job.is_a?(UniqueJob)
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
# frizen_string_literal: true
|
2
|
+
|
3
|
+
require_relative './shared_class_methods'
|
4
|
+
|
5
|
+
module MultiBackgroundJob
|
6
|
+
module Workers
|
7
|
+
module Faktory
|
8
|
+
def self.extended(base)
|
9
|
+
base.include(::Faktory::Job) if defined?(::Faktory)
|
10
|
+
base.extend SharedClassMethods
|
11
|
+
base.extend ClassMethods
|
12
|
+
end
|
13
|
+
|
14
|
+
module ClassMethods
|
15
|
+
def service_worker_options
|
16
|
+
default_queue = MultiBackgroundJob.config.workers.dig(self.name, :queue)
|
17
|
+
default_retry = MultiBackgroundJob.config.workers.dig(self.name, :retry)
|
18
|
+
default_queue ||= ::Faktory.default_job_options['queue'] if defined?(::Faktory)
|
19
|
+
default_retry ||= ::Faktory.default_job_options['retry'] if defined?(::Faktory)
|
20
|
+
{
|
21
|
+
queue: (default_queue || 'default'),
|
22
|
+
retry: (default_retry || 25),
|
23
|
+
}
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# frizen_string_literal: true
|
2
|
+
|
3
|
+
module MultiBackgroundJob
|
4
|
+
module Workers
|
5
|
+
module SharedClassMethods
|
6
|
+
def perform_async(*args)
|
7
|
+
build_worker.with_args(*args).push
|
8
|
+
end
|
9
|
+
|
10
|
+
def perform_in(interval, *args)
|
11
|
+
build_worker.with_args(*args).at(interval).push
|
12
|
+
end
|
13
|
+
alias_method :perform_at, :perform_in
|
14
|
+
|
15
|
+
protected
|
16
|
+
|
17
|
+
def service_worker_options
|
18
|
+
{}
|
19
|
+
end
|
20
|
+
|
21
|
+
def build_worker
|
22
|
+
MultiBackgroundJob[self.name, **service_worker_options.merge(bg_worker_options)]
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|