multi-background-job 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,41 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Server midleware for Faktory
4
+ #
5
+ # @see https://github.com/contribsys/faktory_worker_ruby/wiki/Middleware
6
+ module MultiBackgroundJob
7
+ module Middleware
8
+ class UniqueJob
9
+ module Faktory
10
+ def self.bootstrap
11
+ if defined?(::Faktory)
12
+ ::Faktory.configure_worker do |config|
13
+ config.worker_middleware do |chain|
14
+ chain.add Worker
15
+ end
16
+ end
17
+ end
18
+ end
19
+
20
+ # Worker middleware runs around the execution of a job
21
+ class Worker
22
+ def call(_jobinst, payload)
23
+ if payload.is_a?(Hash) && (unique_lock = unique_job_lock(payload))
24
+ unique_lock.unlock
25
+ end
26
+ yield
27
+ end
28
+
29
+ protected
30
+
31
+ def unique_job_lock(payload)
32
+ return unless payload['uniq'].is_a?(Hash)
33
+
34
+ unique_job = ::MultiBackgroundJob::UniqueJob.coerce(payload['uniq'])
35
+ unique_job&.lock
36
+ end
37
+ end
38
+ end
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,48 @@
1
+ # frozen_string_literal: true
2
+
3
+
4
+ # Provides the Sidekiq middleware that make the unique job control work
5
+ #
6
+ # @see https://github.com/contribsys/faktory_worker_ruby/wiki/Middleware
7
+ module MultiBackgroundJob
8
+ module Middleware
9
+ class UniqueJob
10
+ module Sidekiq
11
+ def self.bootstrap
12
+ if defined?(::Sidekiq)
13
+ ::Sidekiq.configure_worker do |config|
14
+ config.worker_middleware do |chain|
15
+ chain.add Worker
16
+ end
17
+ end
18
+ end
19
+ end
20
+
21
+ # Worker middleware runs around the execution of a job
22
+ class Worker
23
+ # @param worker [Object] the worker instance
24
+ # @param job [Hash] the full job payload
25
+ # * @see https://github.com/mperham/sidekiq/wiki/Job-Format
26
+ # @param queue [String] the name of the queue the job was pulled from
27
+ # @yield the next middleware in the chain or worker `perform` method
28
+ # @return [Void]
29
+ def call(_worker, job, _queue)
30
+ if job.is_a?(Hash) && (unique_lock = unique_job_lock(job))
31
+ unique_lock.unlock
32
+ end
33
+ yield
34
+ end
35
+
36
+ protected
37
+
38
+ def unique_job_lock(job)
39
+ return unless job['uniq'].is_a?(Hash)
40
+
41
+ unique_job = ::MultiBackgroundJob::UniqueJob.coerce(job['uniq'])
42
+ unique_job&.lock
43
+ end
44
+ end
45
+ end
46
+ end
47
+ end
48
+ end
@@ -0,0 +1,109 @@
1
+ # frozen_string_literal: true
2
+
3
+ module MultiBackgroundJob
4
+ # Middleware is code configured to run before/after push a new job.
5
+ # It is patterned after Rack middleware for some modification before push the job to the server
6
+ #
7
+ # To add a middleware:
8
+ #
9
+ # MultiBackgroundJob.configure do |config|
10
+ # config.middleware do |chain|
11
+ # chain.add MyMiddleware
12
+ # end
13
+ # end
14
+ #
15
+ # This is an example of a minimal middleware, note the method must return the result
16
+ # or the job will not push the server.
17
+ #
18
+ # class MyMiddleware
19
+ # def call(job, conn_pool)
20
+ # puts "Before push"
21
+ # result = yield
22
+ # puts "After push"
23
+ # result
24
+ # end
25
+ # end
26
+ #
27
+ class MiddlewareChain
28
+ include Enumerable
29
+ attr_reader :entries
30
+
31
+ def initialize_copy(copy)
32
+ copy.instance_variable_set(:@entries, entries.dup)
33
+ end
34
+
35
+ def each(&block)
36
+ entries.each(&block)
37
+ end
38
+
39
+ def initialize
40
+ @entries = []
41
+ yield self if block_given?
42
+ end
43
+
44
+ def remove(klass)
45
+ entries.delete_if { |entry| entry.klass == klass }
46
+ end
47
+
48
+ def add(klass, *args)
49
+ remove(klass) if exists?(klass)
50
+ entries << Entry.new(klass, *args)
51
+ end
52
+
53
+ def prepend(klass, *args)
54
+ remove(klass) if exists?(klass)
55
+ entries.insert(0, Entry.new(klass, *args))
56
+ end
57
+
58
+ def insert_before(oldklass, newklass, *args)
59
+ i = entries.index { |entry| entry.klass == newklass }
60
+ new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
61
+ i = entries.index { |entry| entry.klass == oldklass } || 0
62
+ entries.insert(i, new_entry)
63
+ end
64
+
65
+ def insert_after(oldklass, newklass, *args)
66
+ i = entries.index { |entry| entry.klass == newklass }
67
+ new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
68
+ i = entries.index { |entry| entry.klass == oldklass } || entries.count - 1
69
+ entries.insert(i+1, new_entry)
70
+ end
71
+
72
+ def exists?(klass)
73
+ any? { |entry| entry.klass == klass }
74
+ end
75
+
76
+ def retrieve
77
+ map(&:make_new)
78
+ end
79
+
80
+ def clear
81
+ entries.clear
82
+ end
83
+
84
+ def invoke(*args)
85
+ chain = retrieve.dup
86
+ traverse_chain = lambda do
87
+ if chain.empty?
88
+ yield
89
+ else
90
+ chain.pop.call(*args, &traverse_chain)
91
+ end
92
+ end
93
+ traverse_chain.call
94
+ end
95
+
96
+ class Entry
97
+ attr_reader :klass, :args
98
+
99
+ def initialize(klass, *args)
100
+ @klass = klass
101
+ @args = args
102
+ end
103
+
104
+ def make_new
105
+ @klass.new(*@args)
106
+ end
107
+ end
108
+ end
109
+ end
@@ -0,0 +1,84 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative './lock'
4
+
5
+ module MultiBackgroundJob
6
+ class UniqueJob
7
+ VALID_OPTIONS = {
8
+ across: %i[queue systemwide],
9
+ timeout: 604800, # 1 week
10
+ unlock_policy: %i[success start],
11
+ }.freeze
12
+
13
+ attr_reader :across, :timeout, :unlock_policy, :lock
14
+
15
+ # @options [Hash] Unique definitions
16
+ # @option [Symbol] :across Valid options are :queue and :systemwide. If jobs should not to be duplicated on
17
+ # current queue or the entire system
18
+ # @option [Integer] :timeout Amount of times in seconds. Timeout decides how long to wait for acquiring the lock.
19
+ # A default timeout is defined to 1 week so unique locks won't last forever.
20
+ # @option [Symbol] :unlock_policy Control when the unique lock is removed. The default value is `success`.
21
+ # The job will not unlock until it executes successfully, it will remain locked even if it raises an error and
22
+ # goes into the retry queue. The alternative value is `start` the job will unlock right before it starts executing
23
+ def initialize(across: :queue, timeout: nil, unlock_policy: :success, lock: nil)
24
+ unless VALID_OPTIONS[:across].include?(across.to_sym)
25
+ raise Error, format('Invalid `across: %<given>p` option. Only %<expected>p are allowed.',
26
+ given: across,
27
+ expected: VALID_OPTIONS[:across])
28
+ end
29
+ unless VALID_OPTIONS[:unlock_policy].include?(unlock_policy.to_sym)
30
+ raise Error, format('Invalid `unlock_policy: %<given>p` option. Only %<expected>p are allowed.',
31
+ given: unlock_policy,
32
+ expected: VALID_OPTIONS[:unlock_policy])
33
+ end
34
+ timeout = VALID_OPTIONS[:timeout] if timeout.to_i <= 0
35
+
36
+ @across = across.to_sym
37
+ @timeout = timeout.to_i
38
+ @unlock_policy = unlock_policy.to_sym
39
+ @lock = lock if lock.is_a?(MultiBackgroundJob::Lock)
40
+ end
41
+
42
+ def self.coerce(value)
43
+ return unless value.is_a?(Hash)
44
+
45
+ new(
46
+ across: (value['across'] || value[:across] || :queue).to_sym,
47
+ timeout: (value['timeout'] || value[:timeout] || nil),
48
+ unlock_policy: (value['unlock_policy'] || value[:unlock_policy] || :success).to_sym,
49
+ ).tap do |instance|
50
+ instance.lock = value['lock'] || value[:lock]
51
+ end
52
+ end
53
+
54
+ def ttl
55
+ Time.now.to_f + timeout
56
+ end
57
+
58
+ def to_hash
59
+ {
60
+ 'across' => (across.to_s if across),
61
+ 'timeout' => timeout,
62
+ 'unlock_policy' => (unlock_policy.to_s if unlock_policy),
63
+ }.tap do |hash|
64
+ hash['lock'] = lock.to_hash if lock
65
+ end
66
+ end
67
+
68
+ def eql?(other)
69
+ return false unless other.is_a?(self.class)
70
+
71
+ [across, timeout, unlock_policy] == [other.across, other.timeout, other.unlock_policy]
72
+ end
73
+ alias == eql?
74
+
75
+ def lock=(value)
76
+ @lock = case value
77
+ when MultiBackgroundJob::Lock then value
78
+ when Hash then MultiBackgroundJob::Lock.coerce(value)
79
+ else
80
+ nil
81
+ end
82
+ end
83
+ end
84
+ end
@@ -0,0 +1,5 @@
1
+ # frozen_string_literal: true
2
+
3
+ module MultiBackgroundJob
4
+ VERSION = '0.1.0'
5
+ end
@@ -0,0 +1,114 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative './unique_job'
4
+
5
+ module MultiBackgroundJob
6
+ class Worker
7
+ attr_reader :options, :payload, :worker_class, :unique_job
8
+
9
+ attr_reader :arguments
10
+
11
+ def initialize(worker_class, **options)
12
+ @worker_class = worker_class
13
+ @options = options
14
+ @payload = {}
15
+ unique(@options.delete(:uniq)) if @options.key?(:uniq)
16
+ end
17
+
18
+ def self.coerce(service:, payload:, **opts)
19
+ SERVICES.fetch(service).coerce_to_worker(payload, **opts)
20
+ end
21
+
22
+ %i[created_at enqueued_at].each do |method_name|
23
+ define_method method_name do |value|
24
+ @payload[method_name.to_s] = \
25
+ case value
26
+ when Numeric then value.to_f
27
+ when String then Time.parse(value).to_f
28
+ when Time, DateTime then value.to_f
29
+ else
30
+ raise ArgumentError, format('The %<v>p is not a valid value for %<m>s.', v: value, m: method_name)
31
+ end
32
+
33
+ self
34
+ end
35
+ end
36
+
37
+ # Adds arguments to the job
38
+ # @return self
39
+ def with_args(*args)
40
+ @payload['args'] = args
41
+
42
+ self
43
+ end
44
+
45
+ # Schedule the time when a job will be executed. Jobs which are scheduled in the past are enqueued for immediate execution.
46
+ # @param timestamp [Numeric] timestamp, numeric or something that acts numeric.
47
+ # @return self
48
+ def in(timestamp)
49
+ now = Time.now.to_f
50
+ timestamp = Time.parse(timestamp) if timestamp.is_a?(String)
51
+ int = timestamp.respond_to?(:strftime) ? timestamp.to_f : now + timestamp.to_f
52
+ return self if int <= now
53
+
54
+ @payload['at'] = int
55
+ @payload['created_at'] = now
56
+
57
+ self
58
+ end
59
+ alias_method :at, :in
60
+
61
+ # Wrap uniq options
62
+ #
63
+ # @param value [Hash] Unique configurations with `across`, `timeout` and `unlock_policy`
64
+ # @return self
65
+ def unique(value)
66
+ value = {} if value == true
67
+ @unique_job = \
68
+ case value
69
+ when Hash then UniqueJob.coerce(value)
70
+ when UniqueJob then value
71
+ else
72
+ nil
73
+ end
74
+
75
+ self
76
+ end
77
+
78
+ def with_job_jid(jid = nil)
79
+ @payload['jid'] ||= jid || MultiBackgroundJob.jid
80
+
81
+ self
82
+ end
83
+
84
+ # @param :to [Symbol] Adapter key
85
+ # @return Response of service
86
+ # @see MultiBackgroundJob::Adapters::** for more details
87
+ def push(to: nil)
88
+ to ||= options[:service]
89
+ unless SERVICES.include?(to)
90
+ raise Error, format('Service %<to>p is not implemented. Please use one of %<list>p', to: to, list: SERVICES.keys)
91
+ end
92
+
93
+ @payload['created_at'] ||= Time.now.to_f
94
+ worker_to_push = with_job_jid
95
+ MultiBackgroundJob.config.middleware.invoke(worker_to_push, to) do
96
+ SERVICES[to].push(worker_to_push)
97
+ end
98
+ end
99
+
100
+ def eql?(other)
101
+ return false unless other.is_a?(self.class)
102
+
103
+ worker_class == other.worker_class && \
104
+ payload == other.payload &&
105
+ options == other.options &&
106
+ unique_job == other.unique_job
107
+ end
108
+ alias == eql?
109
+
110
+ def unique_job?
111
+ unique_job.is_a?(UniqueJob)
112
+ end
113
+ end
114
+ end
@@ -0,0 +1,28 @@
1
+ # frizen_string_literal: true
2
+
3
+ require_relative './shared_class_methods'
4
+
5
+ module MultiBackgroundJob
6
+ module Workers
7
+ module Faktory
8
+ def self.extended(base)
9
+ base.include(::Faktory::Job) if defined?(::Faktory)
10
+ base.extend SharedClassMethods
11
+ base.extend ClassMethods
12
+ end
13
+
14
+ module ClassMethods
15
+ def service_worker_options
16
+ default_queue = MultiBackgroundJob.config.workers.dig(self.name, :queue)
17
+ default_retry = MultiBackgroundJob.config.workers.dig(self.name, :retry)
18
+ default_queue ||= ::Faktory.default_job_options['queue'] if defined?(::Faktory)
19
+ default_retry ||= ::Faktory.default_job_options['retry'] if defined?(::Faktory)
20
+ {
21
+ queue: (default_queue || 'default'),
22
+ retry: (default_retry || 25),
23
+ }
24
+ end
25
+ end
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,26 @@
1
+ # frizen_string_literal: true
2
+
3
+ module MultiBackgroundJob
4
+ module Workers
5
+ module SharedClassMethods
6
+ def perform_async(*args)
7
+ build_worker.with_args(*args).push
8
+ end
9
+
10
+ def perform_in(interval, *args)
11
+ build_worker.with_args(*args).at(interval).push
12
+ end
13
+ alias_method :perform_at, :perform_in
14
+
15
+ protected
16
+
17
+ def service_worker_options
18
+ {}
19
+ end
20
+
21
+ def build_worker
22
+ MultiBackgroundJob[self.name, **service_worker_options.merge(bg_worker_options)]
23
+ end
24
+ end
25
+ end
26
+ end