exekutor 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- checksums.yaml.gz.sig +3 -0
- data/LICENSE.txt +21 -0
- data/exe/exekutor +7 -0
- data/lib/active_job/queue_adapters/exekutor_adapter.rb +14 -0
- data/lib/exekutor/asynchronous.rb +188 -0
- data/lib/exekutor/cleanup.rb +56 -0
- data/lib/exekutor/configuration.rb +373 -0
- data/lib/exekutor/hook.rb +172 -0
- data/lib/exekutor/info/worker.rb +20 -0
- data/lib/exekutor/internal/base_record.rb +11 -0
- data/lib/exekutor/internal/callbacks.rb +138 -0
- data/lib/exekutor/internal/cli/app.rb +173 -0
- data/lib/exekutor/internal/cli/application_loader.rb +36 -0
- data/lib/exekutor/internal/cli/cleanup.rb +96 -0
- data/lib/exekutor/internal/cli/daemon.rb +108 -0
- data/lib/exekutor/internal/cli/default_option_value.rb +29 -0
- data/lib/exekutor/internal/cli/info.rb +126 -0
- data/lib/exekutor/internal/cli/manager.rb +260 -0
- data/lib/exekutor/internal/configuration_builder.rb +113 -0
- data/lib/exekutor/internal/database_connection.rb +21 -0
- data/lib/exekutor/internal/executable.rb +75 -0
- data/lib/exekutor/internal/executor.rb +242 -0
- data/lib/exekutor/internal/hooks.rb +87 -0
- data/lib/exekutor/internal/listener.rb +176 -0
- data/lib/exekutor/internal/logger.rb +74 -0
- data/lib/exekutor/internal/provider.rb +308 -0
- data/lib/exekutor/internal/reserver.rb +95 -0
- data/lib/exekutor/internal/status_server.rb +132 -0
- data/lib/exekutor/job.rb +31 -0
- data/lib/exekutor/job_error.rb +11 -0
- data/lib/exekutor/job_options.rb +95 -0
- data/lib/exekutor/plugins/appsignal.rb +46 -0
- data/lib/exekutor/plugins.rb +13 -0
- data/lib/exekutor/queue.rb +141 -0
- data/lib/exekutor/version.rb +6 -0
- data/lib/exekutor/worker.rb +219 -0
- data/lib/exekutor.rb +49 -0
- data/lib/generators/exekutor/configuration_generator.rb +18 -0
- data/lib/generators/exekutor/install_generator.rb +43 -0
- data/lib/generators/exekutor/templates/install/functions/job_notifier.sql +7 -0
- data/lib/generators/exekutor/templates/install/functions/requeue_orphaned_jobs.sql +7 -0
- data/lib/generators/exekutor/templates/install/initializers/exekutor.rb.erb +14 -0
- data/lib/generators/exekutor/templates/install/migrations/create_exekutor_schema.rb.erb +83 -0
- data/lib/generators/exekutor/templates/install/triggers/notify_workers.sql +6 -0
- data/lib/generators/exekutor/templates/install/triggers/requeue_orphaned_jobs.sql +5 -0
- data.tar.gz.sig +0 -0
- metadata +403 -0
- metadata.gz.sig +0 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 5bc7c7a8f6af32b1f764124d3905673240c167abd19b93e492016433414078a9
|
4
|
+
data.tar.gz: 37f8fb516c32a2a71595b2d608b2610ba78a5396e7f1b1e6bc7c2e90da8d770d
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 8f36b7e5a5968009ba7f33657a0444053a72a5e7b91ebb8e5605c435d80a6861eec4a2d39af81c71ec6e88564a57f7d91fdaca43182ca9e5faf8c86cdd5ab7ff
|
7
|
+
data.tar.gz: ab262caf9827f3eccd5995309b8f0debc0da125af459b8a8321fbd43c6a25d9bcd42dc6fa65f5a98bbb341be07fca4251a8ab73860d01c2ce43efd388499a145
|
checksums.yaml.gz.sig
ADDED
@@ -0,0 +1,3 @@
|
|
1
|
+
e.o����+s�9�=S�����y��C����[6*�s���>��Wu ��._���q���_��.�No�57Z�U�t�'kj}�V�l���9h�?���@ЋYݭJ�nW
|
2
|
+
���a ��XF�B#�]6|���R�!�YҚ��Ʃ��0P��FG�"q��2�{<���b��W{��^��gAm034R/��q�f.r��"�:�I��h0�>�|y�ry�g1r�8�^�#ۆ�7�/�W����G�v�1+���F�q�<�����y%��7Ҁ��9�sc�jc�0�Z����Ϯ��LP��%�
|
3
|
+
��;g\Rl��g���K_�1W�-J����YՃXz��js;���|����_���4{ B�WV�sڟ6 �7vʇ�b4�
|
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2022 Roy
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/exe/exekutor
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require "active_job"
|
3
|
+
require "active_job/queue_adapters"
|
4
|
+
|
5
|
+
module ActiveJob
|
6
|
+
module QueueAdapters
|
7
|
+
# The active job queue adapter for Exekutor
|
8
|
+
class ExekutorAdapter < Exekutor::Queue
|
9
|
+
alias enqueue push
|
10
|
+
alias enqueue_all push
|
11
|
+
alias enqueue_at schedule_at
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,188 @@
|
|
1
|
+
module Exekutor
|
2
|
+
# Mixin to let methods be executed asynchronously by active job
|
3
|
+
#
|
4
|
+
# @example Mark methods as asynchronous
|
5
|
+
# class MyClass
|
6
|
+
# include Exekutor::Asynchronous
|
7
|
+
#
|
8
|
+
# def instance_method
|
9
|
+
# puts "This will be performed by an Exekutor worker"
|
10
|
+
# end
|
11
|
+
# perform_asynchronously :instance_method
|
12
|
+
#
|
13
|
+
# def self.class_method(str)
|
14
|
+
# puts "This will also be performed by an Exekutor worker: #{str}"
|
15
|
+
# end
|
16
|
+
# perform_asynchronously :class_method, class_method: true
|
17
|
+
# end
|
18
|
+
module Asynchronous
|
19
|
+
extend ActiveSupport::Concern
|
20
|
+
|
21
|
+
included do
|
22
|
+
mattr_reader :__async_class_methods, instance_accessor: false, default: {}
|
23
|
+
mattr_reader :__async_instance_methods, instance_accessor: false, default: {}
|
24
|
+
private_class_method :perform_asynchronously
|
25
|
+
end
|
26
|
+
|
27
|
+
class_methods do
|
28
|
+
# Changes a method to be executed asynchronously.
|
29
|
+
# Be aware that you can no longer use the return value for
|
30
|
+
# asynchronous methods, because the actual method will be performed by a worker at a later time. The new
|
31
|
+
# implementation of the method will always return an instance of {AsyncMethodJob}.
|
32
|
+
# If the method takes parameters they must be serializable by active job, otherwise an
|
33
|
+
# +ActiveJob::SerializationError+ will be raised.
|
34
|
+
# @param method_name [Symbol] the method to be executed asynchronous
|
35
|
+
# @param alias_to [String] specifies the new name for the synchronous method
|
36
|
+
# @param class_method [Boolean] whether the method is a class method.
|
37
|
+
# @raise [Error] if the method could not be replaced with the asynchronous version
|
38
|
+
def perform_asynchronously(method_name, alias_to: "__immediately_#{method_name}", class_method: false)
|
39
|
+
raise ArgumentError, "method_name must be a Symbol (actual: #{method_name.class.name})" unless method_name.is_a? Symbol
|
40
|
+
raise ArgumentError, "alias_to must be present" unless alias_to.present?
|
41
|
+
if class_method
|
42
|
+
raise ArgumentError, "##{method_name} does not exist" unless respond_to? method_name, true
|
43
|
+
|
44
|
+
delegate = singleton_class
|
45
|
+
definitions = __async_class_methods
|
46
|
+
else
|
47
|
+
unless method_defined?(method_name, true) || private_method_defined?(method_name, true)
|
48
|
+
raise ArgumentError, "##{method_name} does not exist"
|
49
|
+
end
|
50
|
+
|
51
|
+
delegate = self
|
52
|
+
definitions = __async_instance_methods
|
53
|
+
end
|
54
|
+
if definitions.include? method_name
|
55
|
+
raise Error, "##{method_name} was already marked as asynchronous"
|
56
|
+
end
|
57
|
+
|
58
|
+
delegate.alias_method alias_to, method_name
|
59
|
+
delegate.define_method method_name do |*args, **kwargs|
|
60
|
+
error = Asynchronous.validate_args(self, alias_to, *args, **kwargs)
|
61
|
+
raise error if error
|
62
|
+
raise ArgumentError, "Cannot asynchronously execute with a block argument" if block_given?
|
63
|
+
AsyncMethodJob.perform_later self, method_name, [args, kwargs.presence]
|
64
|
+
end
|
65
|
+
|
66
|
+
definitions[method_name] = alias_to
|
67
|
+
if delegate.public_method_defined?(alias_to)
|
68
|
+
delegate.send :public, method_name
|
69
|
+
elsif delegate.protected_method_defined?(alias_to)
|
70
|
+
delegate.send :protected, method_name
|
71
|
+
else
|
72
|
+
delegate.send :private, method_name
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
private
|
78
|
+
|
79
|
+
# Validates whether the given arguments match the expected parameters for +method+
|
80
|
+
# @param delegate [Object] the object the +method+ will be called on
|
81
|
+
# @param method [Symbol] the method that will be called on +delegate+
|
82
|
+
# @param args [Array] the arguments that will be given to the method
|
83
|
+
# @param kwargs [Hash] the keyword arguments that will be given to the method
|
84
|
+
# @return [ArgumentError,nil] nil if the keywords are valid; an ArgumentError otherwise
|
85
|
+
def self.validate_args(delegate, method, *args, **kwargs)
|
86
|
+
obj_method = delegate.method(method)
|
87
|
+
min_arg_length = 0
|
88
|
+
max_arg_length = 0
|
89
|
+
accepts_keywords = false
|
90
|
+
missing_keywords = []
|
91
|
+
unknown_keywords = kwargs.keys
|
92
|
+
obj_method.parameters.each do |type, name|
|
93
|
+
if type == :req
|
94
|
+
min_arg_length += 1
|
95
|
+
max_arg_length += 1 if max_arg_length
|
96
|
+
elsif type == :opt
|
97
|
+
max_arg_length += 1 if max_arg_length
|
98
|
+
elsif type == :rest
|
99
|
+
max_arg_length = nil
|
100
|
+
elsif type == :keyreq
|
101
|
+
accepts_keywords = true
|
102
|
+
missing_keywords << name if kwargs.exclude?(name)
|
103
|
+
unknown_keywords.delete(name)
|
104
|
+
elsif type == :key
|
105
|
+
accepts_keywords = true
|
106
|
+
unknown_keywords.delete(name)
|
107
|
+
elsif type == :keyrest
|
108
|
+
accepts_keywords = true
|
109
|
+
unknown_keywords = []
|
110
|
+
end
|
111
|
+
end
|
112
|
+
if missing_keywords.present?
|
113
|
+
return ArgumentError.new "missing keyword#{"s" if missing_keywords.many?}: #{missing_keywords.map(&:inspect).join(", ")}"
|
114
|
+
end
|
115
|
+
if accepts_keywords
|
116
|
+
if unknown_keywords.present?
|
117
|
+
return ArgumentError.new "unknown keyword#{"s" if unknown_keywords.many?}: #{unknown_keywords.map(&:inspect).join(", ")}"
|
118
|
+
end
|
119
|
+
elsif kwargs.present?
|
120
|
+
args += [kwargs]
|
121
|
+
end
|
122
|
+
|
123
|
+
args_len = args.length
|
124
|
+
if min_arg_length > args_len || (max_arg_length.present? && max_arg_length < args_len)
|
125
|
+
expected = min_arg_length.to_s
|
126
|
+
if max_arg_length.nil?
|
127
|
+
expected += "+"
|
128
|
+
elsif max_arg_length > min_arg_length
|
129
|
+
expected += "..#{max_arg_length}"
|
130
|
+
end
|
131
|
+
return ArgumentError.new "wrong number of arguments (given #{args_len}, expected #{expected})"
|
132
|
+
end
|
133
|
+
end
|
134
|
+
|
135
|
+
# The internal job used for {Exekutor::Asynchronous}. Only works for methods that are marked as asynchronous to
|
136
|
+
# prevent remote code execution. Include the {Exekutor::Asynchronous} and call
|
137
|
+
# {Exekutor::Asynchronous#perform_asynchronously} to mark a method as asynchronous.
|
138
|
+
class AsyncMethodJob < ActiveJob::Base
|
139
|
+
|
140
|
+
# Calls the original, synchronous method
|
141
|
+
# @!visibility private
|
142
|
+
def perform(object, method, args)
|
143
|
+
check_object! object
|
144
|
+
method_alias = check_method! object, method
|
145
|
+
args, kwargs = args
|
146
|
+
if kwargs
|
147
|
+
object.__send__(method_alias, *args, **kwargs)
|
148
|
+
else
|
149
|
+
object.__send__(method_alias, *args)
|
150
|
+
end
|
151
|
+
end
|
152
|
+
|
153
|
+
private
|
154
|
+
|
155
|
+
def check_object!(object)
|
156
|
+
if object.nil?
|
157
|
+
raise Error, "Object cannot be nil"
|
158
|
+
elsif object.is_a? Class
|
159
|
+
unless object.included_modules.include? Asynchronous
|
160
|
+
raise Error, "Object has not included Exekutor::Asynchronous"
|
161
|
+
end
|
162
|
+
else
|
163
|
+
raise Error, "Object has not included Exekutor::Asynchronous" unless object.is_a? Asynchronous
|
164
|
+
end
|
165
|
+
end
|
166
|
+
|
167
|
+
def check_method!(object, method)
|
168
|
+
if object.is_a? Class
|
169
|
+
class_name = object.name
|
170
|
+
definitions = object.__async_class_methods
|
171
|
+
else
|
172
|
+
class_name = object.class.name
|
173
|
+
definitions = object.class.__async_instance_methods
|
174
|
+
end
|
175
|
+
unless object.respond_to? method, true
|
176
|
+
raise Error, "#{class_name} does not respond to #{method}"
|
177
|
+
end
|
178
|
+
unless definitions.include? method.to_sym
|
179
|
+
raise Error, "#{class_name}##{method} is not marked as asynchronous"
|
180
|
+
end
|
181
|
+
definitions[method.to_sym]
|
182
|
+
end
|
183
|
+
end
|
184
|
+
|
185
|
+
# Raised when an error occurs while configuring or executing asynchronous methods
|
186
|
+
class Error < Exekutor::DiscardJob; end
|
187
|
+
end
|
188
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
module Exekutor
|
2
|
+
# Helper class to clean up finished jobs and stale workers.
|
3
|
+
class Cleanup
|
4
|
+
|
5
|
+
# Purges all workers where the last heartbeat is over the +timeout+ ago.
|
6
|
+
# @param timeout [ActiveSupport::Duration,Numeric,Time] the timeout. Default: 4 hours
|
7
|
+
# @return [Array<Exekutor::Info::Worker>] the purged workers
|
8
|
+
def cleanup_workers(timeout: 4.hours)
|
9
|
+
destroy_before = case timeout
|
10
|
+
when ActiveSupport::Duration
|
11
|
+
timeout.ago
|
12
|
+
when Numeric
|
13
|
+
timeout.hours.ago
|
14
|
+
when Date, Time
|
15
|
+
timeout
|
16
|
+
else
|
17
|
+
raise ArgumentError, "Unsupported value for timeout: #{timeout.class}"
|
18
|
+
end
|
19
|
+
# TODO PG-NOTIFY each worker with an EXIT command
|
20
|
+
Exekutor::Info::Worker.where(%{"last_heartbeat_at"<?}, destroy_before).destroy_all
|
21
|
+
end
|
22
|
+
|
23
|
+
# Purges all jobs where scheduled at is before +before+. Only purges jobs with the given status, if no status is
|
24
|
+
# given all jobs that are not pending are purged.
|
25
|
+
# @param before [ActiveSupport::Duration,Numeric,Time] the maximum scheduled at. Default: 48 hours ago
|
26
|
+
# @param status [Array<String,Symbol>,String,Symbol] the statuses to purge. Default: All except +:pending+
|
27
|
+
# @return [Integer] the number of purged jobs
|
28
|
+
def cleanup_jobs(before: 48.hours.ago, status: nil)
|
29
|
+
destroy_before = case before
|
30
|
+
when ActiveSupport::Duration
|
31
|
+
before.ago
|
32
|
+
when Numeric
|
33
|
+
before.hours.ago
|
34
|
+
when Date, Time
|
35
|
+
before
|
36
|
+
else
|
37
|
+
raise ArgumentError, "Unsupported value for before: #{before.class}"
|
38
|
+
end
|
39
|
+
unless [Array, String, Symbol, NilClass].any?(&status.method(:is_a?))
|
40
|
+
raise ArgumentError, "Unsupported value for status: #{status.class}"
|
41
|
+
end
|
42
|
+
|
43
|
+
jobs = Exekutor::Job.all
|
44
|
+
unless before.nil?
|
45
|
+
jobs.where!(%{"scheduled_at"<?}, destroy_before)
|
46
|
+
end
|
47
|
+
if status
|
48
|
+
jobs.where! status: status
|
49
|
+
else
|
50
|
+
jobs = jobs.where.not(status: :p)
|
51
|
+
end
|
52
|
+
jobs.delete_all
|
53
|
+
end
|
54
|
+
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,373 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative "internal/configuration_builder"
|
4
|
+
|
5
|
+
module Exekutor
|
6
|
+
# Configuration for the Exekutor library
|
7
|
+
class Configuration
|
8
|
+
include Internal::ConfigurationBuilder
|
9
|
+
|
10
|
+
# @private
|
11
|
+
DEFAULT_BASE_RECORD_CLASS = "ActiveRecord::Base"
|
12
|
+
private_constant "DEFAULT_BASE_RECORD_CLASS"
|
13
|
+
|
14
|
+
# @!macro
|
15
|
+
# @!method $1
|
16
|
+
# Gets the default queue priority. Is used when enqueueing jobs that don't have a priority set.
|
17
|
+
# === Default value:
|
18
|
+
# 16,383
|
19
|
+
# @return [Integer]
|
20
|
+
# @!method $1=(value)
|
21
|
+
# Sets the default queue priority. Is used when enqueueing jobs that don't have a priority set. Should be
|
22
|
+
# between 1 and 32,767.
|
23
|
+
# @raise [Error] When the priority is nil or invalid
|
24
|
+
# @param value [Integer] the priority
|
25
|
+
# @return [self]
|
26
|
+
define_option :default_queue_priority, default: 16_383, required: true, type: Integer,
|
27
|
+
range: Exekutor::Queue::VALID_PRIORITIES
|
28
|
+
|
29
|
+
# @!macro
|
30
|
+
# @!method $1
|
31
|
+
# Gets the base class name for database records.
|
32
|
+
# === Default value:
|
33
|
+
# +"ActiveRecord::Base"+
|
34
|
+
# @return [String]
|
35
|
+
# @!method $1=(value)
|
36
|
+
# Sets the base class name for database records. The validity of this value will not be checked immediately.
|
37
|
+
# (Ie. When the specified class does not exist, an error will raised when a database record is used for the
|
38
|
+
# first time.)
|
39
|
+
# @raise [Error] When the name is blank
|
40
|
+
# @param value [String] the class name
|
41
|
+
# @return [self]
|
42
|
+
define_option :base_record_class_name, default: DEFAULT_BASE_RECORD_CLASS, required: true, type: String
|
43
|
+
|
44
|
+
# Gets the base class for database records. Is derived from the {#base_record_class_name} option.
|
45
|
+
# @raise [Error] when the class cannot be found
|
46
|
+
# @return [Class]
|
47
|
+
def base_record_class
|
48
|
+
const_get :base_record_class_name
|
49
|
+
rescue ::StandardError
|
50
|
+
# A nicer message for the default value
|
51
|
+
if base_record_class_name == DEFAULT_BASE_RECORD_CLASS
|
52
|
+
raise Error, "Cannot find ActiveRecord, did you install and load the gem?"
|
53
|
+
end
|
54
|
+
|
55
|
+
raise
|
56
|
+
end
|
57
|
+
|
58
|
+
# @!macro
|
59
|
+
# @!method $1
|
60
|
+
# Gets the unconverted JSON serializer value. This can be either a +String+, a +Symbol+, a +Proc+, or the
|
61
|
+
# serializer.
|
62
|
+
# === Default value:
|
63
|
+
# +JSON+
|
64
|
+
# @return [String,Symbol,Proc,Object]
|
65
|
+
# @!method $1=(value)
|
66
|
+
# Sets the JSON serializer. This can be either a +String+, a +Symbol+, a +Proc+, or the serializer. If a
|
67
|
+
# +String+, +Symbol+, or +Proc+ is given, the serializer will be loaded when it is needed for the first time.
|
68
|
+
# If the loaded class does not respond to +#dump+ and +#load+ an {Error} will be raised whenever the serializer
|
69
|
+
# is loaded. If the value is neither a +String+, +Symbol+, nor a +Proc+ and it does not respond to +#dump+ and
|
70
|
+
# +#load+, the error will be thrown immediately.
|
71
|
+
# @raise [Error] When the value is neither a +String+, +Symbol+, nor a +Proc+ and it does not respond to +#dump+
|
72
|
+
# and +#load+
|
73
|
+
# @param value [String,Symbol,Proc,Object] the serializer
|
74
|
+
# @return [self]
|
75
|
+
define_option :json_serializer, default: "::JSON", required: true do |value|
|
76
|
+
unless value.is_a?(String) || value.is_a?(Symbol) || value.respond_to?(:call) ||
|
77
|
+
(value.respond_to?(:dump) && value.respond_to?(:load))
|
78
|
+
raise Error, "#json_serializer must either be a String, a Proc, or respond to #dump and #load"
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
# Gets the JSON serializer. Is derived from the {#json_serializer} option.
|
83
|
+
# @raise [Error] when the class cannot be found, or does not respond to +#dump+ and +#load+
|
84
|
+
# @return [Object]
|
85
|
+
def load_json_serializer
|
86
|
+
raw_value = self.json_serializer
|
87
|
+
if defined?(@json_serializer_instance) && @json_serializer_instance[0] == raw_value
|
88
|
+
return @json_serializer_instance[1]
|
89
|
+
end
|
90
|
+
|
91
|
+
serializer = const_get :json_serializer
|
92
|
+
unless serializer.respond_to?(:dump) && serializer.respond_to?(:load)
|
93
|
+
serializer = serializer.call if serializer.respond_to?(:call)
|
94
|
+
unless serializer.respond_to?(:dump) && serializer.respond_to?(:load)
|
95
|
+
serializer = serializer.new if serializer.respond_to?(:new)
|
96
|
+
end
|
97
|
+
end
|
98
|
+
unless serializer.respond_to?(:dump) && serializer.respond_to?(:load)
|
99
|
+
raise Error, <<~MSG.squish
|
100
|
+
The configured serializer (#{serializer.class}) does not respond to #dump and #load
|
101
|
+
MSG
|
102
|
+
end
|
103
|
+
|
104
|
+
@json_serializer_instance = [raw_value, serializer]
|
105
|
+
serializer
|
106
|
+
end
|
107
|
+
|
108
|
+
# @!macro
|
109
|
+
# @!method $1
|
110
|
+
# Gets the logger.
|
111
|
+
# === Default value:
|
112
|
+
# Rails.active_job.logger
|
113
|
+
# @return [ActiveSupport::Logger]
|
114
|
+
# @!method $1=(value)
|
115
|
+
# Sets the logger.
|
116
|
+
# @param value [ActiveSupport::Logger] the logger
|
117
|
+
# @return [self]
|
118
|
+
define_option :logger, default: -> { Rails.active_job.logger }
|
119
|
+
|
120
|
+
# @!macro
|
121
|
+
# @!method $1?
|
122
|
+
# Whether the DB connection name should be set. Only affects the listener, unless started from the CLI.
|
123
|
+
# === Default value:
|
124
|
+
# false (true when started from the CLI)
|
125
|
+
# @return [Boolean, nil]
|
126
|
+
# @!method $1=(value)
|
127
|
+
# Sets whether the DB connection name should be set
|
128
|
+
# @param value [Boolean] whether to name should be set
|
129
|
+
# @return [self]
|
130
|
+
define_option :set_db_connection_name, type: [TrueClass, FalseClass], required: true
|
131
|
+
|
132
|
+
def set_db_connection_name?
|
133
|
+
if set_db_connection_name.nil?
|
134
|
+
false
|
135
|
+
else
|
136
|
+
set_db_connection_name
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
140
|
+
# @!macro
|
141
|
+
# @!method $1?
|
142
|
+
# Whether the worker should use LISTEN/NOTIFY to listen for jobs.
|
143
|
+
# === Default value:
|
144
|
+
# true
|
145
|
+
# @return [Boolean, nil]
|
146
|
+
# @!method $1=(value)
|
147
|
+
# Sets whether the worker should use LISTEN/NOTIFY to listen for jobs
|
148
|
+
# @param value [Boolean] whether to enable the listener
|
149
|
+
# @return [self]
|
150
|
+
define_option :enable_listener, reader: :enable_listener?, default: true, type: [TrueClass, FalseClass],
|
151
|
+
required: true
|
152
|
+
|
153
|
+
# @!macro
|
154
|
+
# @!method $1?
|
155
|
+
# Whether the worker should delete jobs after completion.
|
156
|
+
# === Default value:
|
157
|
+
# false
|
158
|
+
# @return [Boolean]
|
159
|
+
# @!method $1=(value)
|
160
|
+
# Sets whether the worker should delete jobs after completion
|
161
|
+
# @param value [Boolean] whether to delete completed jobs
|
162
|
+
# @return [self]
|
163
|
+
define_option :delete_completed_jobs, reader: :delete_completed_jobs?, required: true,
|
164
|
+
type: [TrueClass, FalseClass], default: false
|
165
|
+
|
166
|
+
# @!macro
|
167
|
+
# @!method $1?
|
168
|
+
# Whether the worker should delete discarded jobs.
|
169
|
+
# === Default value:
|
170
|
+
# false
|
171
|
+
# @return [Boolean]
|
172
|
+
# @!method $1=(value)
|
173
|
+
# Sets whether the worker should delete discarded jobs
|
174
|
+
# @param value [Boolean] whether to delete discarded jobs
|
175
|
+
# @return [self]
|
176
|
+
define_option :delete_discarded_jobs, reader: :delete_discarded_jobs?, required: true,
|
177
|
+
type: [TrueClass, FalseClass], default: false
|
178
|
+
|
179
|
+
# @!macro
|
180
|
+
# @!method $1?
|
181
|
+
# Whether the worker should delete jobs after they failed to execute.
|
182
|
+
# === Default value:
|
183
|
+
# false
|
184
|
+
# @return [Boolean]
|
185
|
+
# @!method $1=(value)
|
186
|
+
# Sets whether the worker should delete jobs after they failed to execute
|
187
|
+
# @param value [Boolean] whether to delete failed jobs
|
188
|
+
# @return [self]
|
189
|
+
define_option :delete_failed_jobs, reader: :delete_failed_jobs?, required: true,
|
190
|
+
type: [TrueClass, FalseClass], default: false
|
191
|
+
|
192
|
+
# @!macro
|
193
|
+
# @!method $1
|
194
|
+
# The polling interval in seconds. When set, the worker will poll the database with this interval to check for
|
195
|
+
# any pending jobs that a listener might have missed (if enabled).
|
196
|
+
# === Default value:
|
197
|
+
# 60
|
198
|
+
# @return [Integer]
|
199
|
+
# @!method $1=(value)
|
200
|
+
# Sets the polling interval in seconds. Set to +nil+ to disable polling. If the listener is disabled, this value
|
201
|
+
# should be reasonably low so jobs don't have to wait in the queue too long; if it is enabled, this value can
|
202
|
+
# be reasonably high.
|
203
|
+
# @param value [Integer] the interval
|
204
|
+
# @return [self]
|
205
|
+
define_option :polling_interval, default: 60, type: [Integer, nil], range: 1...(1.day.to_i)
|
206
|
+
|
207
|
+
# @!macro
|
208
|
+
# @!method $1
|
209
|
+
# The polling jitter, used to adjust the polling interval slightly so multiple workers will not query the
|
210
|
+
# database at the same time.
|
211
|
+
# === Default value:
|
212
|
+
# 0.1
|
213
|
+
# @return [Float]
|
214
|
+
# @!method $1=(value)
|
215
|
+
# Sets the polling jitter, which is used to slightly adjust the polling interval. Should be between 0 and 0.5.
|
216
|
+
# A value of 0.1 means the polling interval can vary by 10%. If the interval is set to 60 seconds and the jitter
|
217
|
+
# is set to 0.1, the interval can range from 57 to 63 seconds. A value of 0 disables this feature.
|
218
|
+
# @param value [Float] the jitter
|
219
|
+
# @return [self]
|
220
|
+
define_option :polling_jitter, default: 0.1, type: [Float, Integer], range: 0..0.5
|
221
|
+
|
222
|
+
# @!macro
|
223
|
+
# @!method $1
|
224
|
+
# The minimum number of execution threads that should be active.
|
225
|
+
# === Default value:
|
226
|
+
# 1
|
227
|
+
# @return [Integer]
|
228
|
+
# @!method $1=(value)
|
229
|
+
# Sets the minimum number of execution threads that should be active
|
230
|
+
# @param value [Integer] the number of threads
|
231
|
+
# @return [self]
|
232
|
+
define_option :min_execution_threads, default: 1, type: Integer, range: 1...999
|
233
|
+
|
234
|
+
# @!macro
|
235
|
+
# @!method $1
|
236
|
+
# The maximum number of execution threads that may be active.
|
237
|
+
# === Default value:
|
238
|
+
# Active record pool size minus 1, with a minimum of 1
|
239
|
+
# @return [Integer]
|
240
|
+
# @!method $1=(value)
|
241
|
+
# Sets the maximum number of execution threads that may be active. Be aware that if you set this to a value
|
242
|
+
# greater than +connection_db_config.pool+, workers may have to wait for database connections to become
|
243
|
+
# available because all connections are occupied by other threads. This may result in an
|
244
|
+
# +ActiveRecord::ConnectionTimeoutError+ if the thread has to wait too long.
|
245
|
+
# @param value [Integer] the number of threads
|
246
|
+
# @return [self]
|
247
|
+
define_option :max_execution_threads,
|
248
|
+
default: -> { (Internal::BaseRecord.connection_db_config.pool.to_i - 1).clamp(1, 999) },
|
249
|
+
type: Integer, range: 1...999
|
250
|
+
|
251
|
+
# @!macro
|
252
|
+
# @!method $1
|
253
|
+
# The maximum number of seconds a thread may be idle before being stopped.
|
254
|
+
# === Default value:
|
255
|
+
# 60
|
256
|
+
# @return [Integer]
|
257
|
+
# @!method $1=(value)
|
258
|
+
# Sets the maximum number of seconds a thread may be idle before being stopped
|
259
|
+
# @param value [Integer] the number of threads
|
260
|
+
# @return [self]
|
261
|
+
define_option :max_execution_thread_idletime, default: 60, type: Integer, range: 1..(1.day.to_i)
|
262
|
+
|
263
|
+
# @!macro
|
264
|
+
# @!method $1?
|
265
|
+
# The rack handler for the status server
|
266
|
+
# === Default value:
|
267
|
+
# webrick
|
268
|
+
# @return [String]
|
269
|
+
# @!method $1=(value)
|
270
|
+
# Sets the rack handler for the status server. The handler should respond to +#shutdown+ or +#stop+.
|
271
|
+
# @param value [String] the name of the handler
|
272
|
+
# @return [self]
|
273
|
+
define_option :status_server_handler, default: "webrick", type: String
|
274
|
+
|
275
|
+
# @!macro
|
276
|
+
# @!method $1?
|
277
|
+
# The heartbeat timeout for the `/live` endpoint of the status server, in minutes. If the heartbeat of a worker
|
278
|
+
# is older than this timeout, the status server will respond with a 503 status indicating the service is
|
279
|
+
# down.
|
280
|
+
# === Default value:
|
281
|
+
# 30
|
282
|
+
# @return [Integer]
|
283
|
+
# @!method $1=(value)
|
284
|
+
# Sets the heartbeat timeout for the `/live` endpoint of the status server, in minutes. Must be between 2
|
285
|
+
# and 1440 (24 hours).
|
286
|
+
# @param value [Integer] The timeout in minutes
|
287
|
+
# @return [self]
|
288
|
+
define_option :healthcheck_timeout, default: 30, type: Integer, range: 2..1440
|
289
|
+
|
290
|
+
# @!macro
|
291
|
+
# @!method $1?
|
292
|
+
# Whether to suppress STDOUT messages
|
293
|
+
# === Default value:
|
294
|
+
# false
|
295
|
+
# @return [Boolean]
|
296
|
+
# @!method $1=(value)
|
297
|
+
# Sets whether the STDOUT messages should be printed
|
298
|
+
# @param value [Boolean] whether to suppress STDOUT messages
|
299
|
+
# @return [self]
|
300
|
+
define_option :quiet, reader: :quiet?, type: [TrueClass, FalseClass], required: true, default: false
|
301
|
+
|
302
|
+
# Gets the options for a worker
|
303
|
+
# @return [Hash] the worker configuration
|
304
|
+
def worker_options
|
305
|
+
{
|
306
|
+
min_threads: min_execution_threads,
|
307
|
+
max_threads: max_execution_threads,
|
308
|
+
max_thread_idletime: max_execution_thread_idletime,
|
309
|
+
}.tap do |opts|
|
310
|
+
opts[:set_db_connection_name] = set_db_connection_name? unless set_db_connection_name.nil?
|
311
|
+
%i[enable_listener delete_completed_jobs delete_discarded_jobs delete_failed_jobs].each do |option|
|
312
|
+
opts[option] = send(:"#{option}?") ? true : false
|
313
|
+
end
|
314
|
+
%i[polling_interval polling_jitter status_server_handler healthcheck_timeout].each do |option|
|
315
|
+
opts[option] = send(option)
|
316
|
+
end
|
317
|
+
end
|
318
|
+
end
|
319
|
+
|
320
|
+
private
|
321
|
+
|
322
|
+
def const_get(option_name)
|
323
|
+
class_name = send(option_name)
|
324
|
+
case class_name
|
325
|
+
when String, Symbol
|
326
|
+
begin
|
327
|
+
class_name = if class_name.is_a? Symbol
|
328
|
+
class_name.to_s.camelize.prepend("::")
|
329
|
+
elsif class_name.start_with? "::"
|
330
|
+
class_name
|
331
|
+
else
|
332
|
+
class_name.dup.prepend("::")
|
333
|
+
end
|
334
|
+
|
335
|
+
Object.const_get class_name
|
336
|
+
rescue NameError, LoadError
|
337
|
+
raise Error, <<~MSG.squish
|
338
|
+
Cannot convert ##{option_name} (#{class_name.inspect}) to a constant. Have you made a typo?
|
339
|
+
MSG
|
340
|
+
end
|
341
|
+
else
|
342
|
+
class_name
|
343
|
+
end
|
344
|
+
end
|
345
|
+
|
346
|
+
# Raised when configuring an invalid option or value
|
347
|
+
class Error < Exekutor::Error; end
|
348
|
+
|
349
|
+
protected
|
350
|
+
|
351
|
+
def error_class
|
352
|
+
Error
|
353
|
+
end
|
354
|
+
end
|
355
|
+
|
356
|
+
def self.config
|
357
|
+
@config ||= Exekutor::Configuration.new
|
358
|
+
end
|
359
|
+
|
360
|
+
def self.configure(opts = nil, &block)
|
361
|
+
raise ArgumentError, "opts must be a Hash" unless opts.nil? || opts.is_a?(Hash)
|
362
|
+
raise ArgumentError, "Either opts or a block must be given" unless opts.present? || block_given?
|
363
|
+
|
364
|
+
config.set(**opts) if opts
|
365
|
+
return unless block_given?
|
366
|
+
|
367
|
+
if block.arity == 1
|
368
|
+
block.call config
|
369
|
+
else
|
370
|
+
instance_eval(&block)
|
371
|
+
end
|
372
|
+
end
|
373
|
+
end
|