exekutor 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- checksums.yaml.gz.sig +3 -0
- data/LICENSE.txt +21 -0
- data/exe/exekutor +7 -0
- data/lib/active_job/queue_adapters/exekutor_adapter.rb +14 -0
- data/lib/exekutor/asynchronous.rb +188 -0
- data/lib/exekutor/cleanup.rb +56 -0
- data/lib/exekutor/configuration.rb +373 -0
- data/lib/exekutor/hook.rb +172 -0
- data/lib/exekutor/info/worker.rb +20 -0
- data/lib/exekutor/internal/base_record.rb +11 -0
- data/lib/exekutor/internal/callbacks.rb +138 -0
- data/lib/exekutor/internal/cli/app.rb +173 -0
- data/lib/exekutor/internal/cli/application_loader.rb +36 -0
- data/lib/exekutor/internal/cli/cleanup.rb +96 -0
- data/lib/exekutor/internal/cli/daemon.rb +108 -0
- data/lib/exekutor/internal/cli/default_option_value.rb +29 -0
- data/lib/exekutor/internal/cli/info.rb +126 -0
- data/lib/exekutor/internal/cli/manager.rb +260 -0
- data/lib/exekutor/internal/configuration_builder.rb +113 -0
- data/lib/exekutor/internal/database_connection.rb +21 -0
- data/lib/exekutor/internal/executable.rb +75 -0
- data/lib/exekutor/internal/executor.rb +242 -0
- data/lib/exekutor/internal/hooks.rb +87 -0
- data/lib/exekutor/internal/listener.rb +176 -0
- data/lib/exekutor/internal/logger.rb +74 -0
- data/lib/exekutor/internal/provider.rb +308 -0
- data/lib/exekutor/internal/reserver.rb +95 -0
- data/lib/exekutor/internal/status_server.rb +132 -0
- data/lib/exekutor/job.rb +31 -0
- data/lib/exekutor/job_error.rb +11 -0
- data/lib/exekutor/job_options.rb +95 -0
- data/lib/exekutor/plugins/appsignal.rb +46 -0
- data/lib/exekutor/plugins.rb +13 -0
- data/lib/exekutor/queue.rb +141 -0
- data/lib/exekutor/version.rb +6 -0
- data/lib/exekutor/worker.rb +219 -0
- data/lib/exekutor.rb +49 -0
- data/lib/generators/exekutor/configuration_generator.rb +18 -0
- data/lib/generators/exekutor/install_generator.rb +43 -0
- data/lib/generators/exekutor/templates/install/functions/job_notifier.sql +7 -0
- data/lib/generators/exekutor/templates/install/functions/requeue_orphaned_jobs.sql +7 -0
- data/lib/generators/exekutor/templates/install/initializers/exekutor.rb.erb +14 -0
- data/lib/generators/exekutor/templates/install/migrations/create_exekutor_schema.rb.erb +83 -0
- data/lib/generators/exekutor/templates/install/triggers/notify_workers.sql +6 -0
- data/lib/generators/exekutor/templates/install/triggers/requeue_orphaned_jobs.sql +5 -0
- data.tar.gz.sig +0 -0
- metadata +403 -0
- metadata.gz.sig +0 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 5bc7c7a8f6af32b1f764124d3905673240c167abd19b93e492016433414078a9
|
4
|
+
data.tar.gz: 37f8fb516c32a2a71595b2d608b2610ba78a5396e7f1b1e6bc7c2e90da8d770d
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 8f36b7e5a5968009ba7f33657a0444053a72a5e7b91ebb8e5605c435d80a6861eec4a2d39af81c71ec6e88564a57f7d91fdaca43182ca9e5faf8c86cdd5ab7ff
|
7
|
+
data.tar.gz: ab262caf9827f3eccd5995309b8f0debc0da125af459b8a8321fbd43c6a25d9bcd42dc6fa65f5a98bbb341be07fca4251a8ab73860d01c2ce43efd388499a145
|
checksums.yaml.gz.sig
ADDED
@@ -0,0 +1,3 @@
|
|
1
|
+
e.o����+s�9�=S�����y��C����[6*�s���>��Wu ��._���q���_��.�No�57Z�U�t�'kj}�V�l���9h�?���@ЋYݭJ�nW
|
2
|
+
���a ��XF�B#�]6|���R�!�YҚ��Ʃ��0P��FG�"q��2�{<���b��W{��^��gAm034R/��q�f.r��"�:�I��h0�>�|y�ry�g1r�8�^�#ۆ�7�/�W����G�v�1+���F�q�<�����y%��7Ҁ��9�sc�jc�0�Z����Ϯ��LP��%�
|
3
|
+
��;g\Rl��g���K_�1W�-J����YՃXz��js;���|����_���4{ B�WV�sڟ6 �7vʇ�b4�
|
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2022 Roy
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/exe/exekutor
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require "active_job"
|
3
|
+
require "active_job/queue_adapters"
|
4
|
+
|
5
|
+
module ActiveJob
|
6
|
+
module QueueAdapters
|
7
|
+
# The active job queue adapter for Exekutor
|
8
|
+
class ExekutorAdapter < Exekutor::Queue
|
9
|
+
alias enqueue push
|
10
|
+
alias enqueue_all push
|
11
|
+
alias enqueue_at schedule_at
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,188 @@
|
|
1
|
+
module Exekutor
|
2
|
+
# Mixin to let methods be executed asynchronously by active job
|
3
|
+
#
|
4
|
+
# @example Mark methods as asynchronous
|
5
|
+
# class MyClass
|
6
|
+
# include Exekutor::Asynchronous
|
7
|
+
#
|
8
|
+
# def instance_method
|
9
|
+
# puts "This will be performed by an Exekutor worker"
|
10
|
+
# end
|
11
|
+
# perform_asynchronously :instance_method
|
12
|
+
#
|
13
|
+
# def self.class_method(str)
|
14
|
+
# puts "This will also be performed by an Exekutor worker: #{str}"
|
15
|
+
# end
|
16
|
+
# perform_asynchronously :class_method, class_method: true
|
17
|
+
# end
|
18
|
+
module Asynchronous
|
19
|
+
extend ActiveSupport::Concern
|
20
|
+
|
21
|
+
included do
|
22
|
+
mattr_reader :__async_class_methods, instance_accessor: false, default: {}
|
23
|
+
mattr_reader :__async_instance_methods, instance_accessor: false, default: {}
|
24
|
+
private_class_method :perform_asynchronously
|
25
|
+
end
|
26
|
+
|
27
|
+
class_methods do
|
28
|
+
# Changes a method to be executed asynchronously.
|
29
|
+
# Be aware that you can no longer use the return value for
|
30
|
+
# asynchronous methods, because the actual method will be performed by a worker at a later time. The new
|
31
|
+
# implementation of the method will always return an instance of {AsyncMethodJob}.
|
32
|
+
# If the method takes parameters they must be serializable by active job, otherwise an
|
33
|
+
# +ActiveJob::SerializationError+ will be raised.
|
34
|
+
# @param method_name [Symbol] the method to be executed asynchronous
|
35
|
+
# @param alias_to [String] specifies the new name for the synchronous method
|
36
|
+
# @param class_method [Boolean] whether the method is a class method.
|
37
|
+
# @raise [Error] if the method could not be replaced with the asynchronous version
|
38
|
+
def perform_asynchronously(method_name, alias_to: "__immediately_#{method_name}", class_method: false)
|
39
|
+
raise ArgumentError, "method_name must be a Symbol (actual: #{method_name.class.name})" unless method_name.is_a? Symbol
|
40
|
+
raise ArgumentError, "alias_to must be present" unless alias_to.present?
|
41
|
+
if class_method
|
42
|
+
raise ArgumentError, "##{method_name} does not exist" unless respond_to? method_name, true
|
43
|
+
|
44
|
+
delegate = singleton_class
|
45
|
+
definitions = __async_class_methods
|
46
|
+
else
|
47
|
+
unless method_defined?(method_name, true) || private_method_defined?(method_name, true)
|
48
|
+
raise ArgumentError, "##{method_name} does not exist"
|
49
|
+
end
|
50
|
+
|
51
|
+
delegate = self
|
52
|
+
definitions = __async_instance_methods
|
53
|
+
end
|
54
|
+
if definitions.include? method_name
|
55
|
+
raise Error, "##{method_name} was already marked as asynchronous"
|
56
|
+
end
|
57
|
+
|
58
|
+
delegate.alias_method alias_to, method_name
|
59
|
+
delegate.define_method method_name do |*args, **kwargs|
|
60
|
+
error = Asynchronous.validate_args(self, alias_to, *args, **kwargs)
|
61
|
+
raise error if error
|
62
|
+
raise ArgumentError, "Cannot asynchronously execute with a block argument" if block_given?
|
63
|
+
AsyncMethodJob.perform_later self, method_name, [args, kwargs.presence]
|
64
|
+
end
|
65
|
+
|
66
|
+
definitions[method_name] = alias_to
|
67
|
+
if delegate.public_method_defined?(alias_to)
|
68
|
+
delegate.send :public, method_name
|
69
|
+
elsif delegate.protected_method_defined?(alias_to)
|
70
|
+
delegate.send :protected, method_name
|
71
|
+
else
|
72
|
+
delegate.send :private, method_name
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
private
|
78
|
+
|
79
|
+
# Validates whether the given arguments match the expected parameters for +method+
|
80
|
+
# @param delegate [Object] the object the +method+ will be called on
|
81
|
+
# @param method [Symbol] the method that will be called on +delegate+
|
82
|
+
# @param args [Array] the arguments that will be given to the method
|
83
|
+
# @param kwargs [Hash] the keyword arguments that will be given to the method
|
84
|
+
# @return [ArgumentError,nil] nil if the keywords are valid; an ArgumentError otherwise
|
85
|
+
def self.validate_args(delegate, method, *args, **kwargs)
|
86
|
+
obj_method = delegate.method(method)
|
87
|
+
min_arg_length = 0
|
88
|
+
max_arg_length = 0
|
89
|
+
accepts_keywords = false
|
90
|
+
missing_keywords = []
|
91
|
+
unknown_keywords = kwargs.keys
|
92
|
+
obj_method.parameters.each do |type, name|
|
93
|
+
if type == :req
|
94
|
+
min_arg_length += 1
|
95
|
+
max_arg_length += 1 if max_arg_length
|
96
|
+
elsif type == :opt
|
97
|
+
max_arg_length += 1 if max_arg_length
|
98
|
+
elsif type == :rest
|
99
|
+
max_arg_length = nil
|
100
|
+
elsif type == :keyreq
|
101
|
+
accepts_keywords = true
|
102
|
+
missing_keywords << name if kwargs.exclude?(name)
|
103
|
+
unknown_keywords.delete(name)
|
104
|
+
elsif type == :key
|
105
|
+
accepts_keywords = true
|
106
|
+
unknown_keywords.delete(name)
|
107
|
+
elsif type == :keyrest
|
108
|
+
accepts_keywords = true
|
109
|
+
unknown_keywords = []
|
110
|
+
end
|
111
|
+
end
|
112
|
+
if missing_keywords.present?
|
113
|
+
return ArgumentError.new "missing keyword#{"s" if missing_keywords.many?}: #{missing_keywords.map(&:inspect).join(", ")}"
|
114
|
+
end
|
115
|
+
if accepts_keywords
|
116
|
+
if unknown_keywords.present?
|
117
|
+
return ArgumentError.new "unknown keyword#{"s" if unknown_keywords.many?}: #{unknown_keywords.map(&:inspect).join(", ")}"
|
118
|
+
end
|
119
|
+
elsif kwargs.present?
|
120
|
+
args += [kwargs]
|
121
|
+
end
|
122
|
+
|
123
|
+
args_len = args.length
|
124
|
+
if min_arg_length > args_len || (max_arg_length.present? && max_arg_length < args_len)
|
125
|
+
expected = min_arg_length.to_s
|
126
|
+
if max_arg_length.nil?
|
127
|
+
expected += "+"
|
128
|
+
elsif max_arg_length > min_arg_length
|
129
|
+
expected += "..#{max_arg_length}"
|
130
|
+
end
|
131
|
+
return ArgumentError.new "wrong number of arguments (given #{args_len}, expected #{expected})"
|
132
|
+
end
|
133
|
+
end
|
134
|
+
|
135
|
+
# The internal job used for {Exekutor::Asynchronous}. Only works for methods that are marked as asynchronous to
|
136
|
+
# prevent remote code execution. Include the {Exekutor::Asynchronous} and call
|
137
|
+
# {Exekutor::Asynchronous#perform_asynchronously} to mark a method as asynchronous.
|
138
|
+
class AsyncMethodJob < ActiveJob::Base
|
139
|
+
|
140
|
+
# Calls the original, synchronous method
|
141
|
+
# @!visibility private
|
142
|
+
def perform(object, method, args)
|
143
|
+
check_object! object
|
144
|
+
method_alias = check_method! object, method
|
145
|
+
args, kwargs = args
|
146
|
+
if kwargs
|
147
|
+
object.__send__(method_alias, *args, **kwargs)
|
148
|
+
else
|
149
|
+
object.__send__(method_alias, *args)
|
150
|
+
end
|
151
|
+
end
|
152
|
+
|
153
|
+
private
|
154
|
+
|
155
|
+
def check_object!(object)
|
156
|
+
if object.nil?
|
157
|
+
raise Error, "Object cannot be nil"
|
158
|
+
elsif object.is_a? Class
|
159
|
+
unless object.included_modules.include? Asynchronous
|
160
|
+
raise Error, "Object has not included Exekutor::Asynchronous"
|
161
|
+
end
|
162
|
+
else
|
163
|
+
raise Error, "Object has not included Exekutor::Asynchronous" unless object.is_a? Asynchronous
|
164
|
+
end
|
165
|
+
end
|
166
|
+
|
167
|
+
def check_method!(object, method)
|
168
|
+
if object.is_a? Class
|
169
|
+
class_name = object.name
|
170
|
+
definitions = object.__async_class_methods
|
171
|
+
else
|
172
|
+
class_name = object.class.name
|
173
|
+
definitions = object.class.__async_instance_methods
|
174
|
+
end
|
175
|
+
unless object.respond_to? method, true
|
176
|
+
raise Error, "#{class_name} does not respond to #{method}"
|
177
|
+
end
|
178
|
+
unless definitions.include? method.to_sym
|
179
|
+
raise Error, "#{class_name}##{method} is not marked as asynchronous"
|
180
|
+
end
|
181
|
+
definitions[method.to_sym]
|
182
|
+
end
|
183
|
+
end
|
184
|
+
|
185
|
+
# Raised when an error occurs while configuring or executing asynchronous methods
|
186
|
+
class Error < Exekutor::DiscardJob; end
|
187
|
+
end
|
188
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
module Exekutor
|
2
|
+
# Helper class to clean up finished jobs and stale workers.
|
3
|
+
class Cleanup
|
4
|
+
|
5
|
+
# Purges all workers where the last heartbeat is over the +timeout+ ago.
|
6
|
+
# @param timeout [ActiveSupport::Duration,Numeric,Time] the timeout. Default: 4 hours
|
7
|
+
# @return [Array<Exekutor::Info::Worker>] the purged workers
|
8
|
+
def cleanup_workers(timeout: 4.hours)
|
9
|
+
destroy_before = case timeout
|
10
|
+
when ActiveSupport::Duration
|
11
|
+
timeout.ago
|
12
|
+
when Numeric
|
13
|
+
timeout.hours.ago
|
14
|
+
when Date, Time
|
15
|
+
timeout
|
16
|
+
else
|
17
|
+
raise ArgumentError, "Unsupported value for timeout: #{timeout.class}"
|
18
|
+
end
|
19
|
+
# TODO PG-NOTIFY each worker with an EXIT command
|
20
|
+
Exekutor::Info::Worker.where(%{"last_heartbeat_at"<?}, destroy_before).destroy_all
|
21
|
+
end
|
22
|
+
|
23
|
+
# Purges all jobs where scheduled at is before +before+. Only purges jobs with the given status, if no status is
|
24
|
+
# given all jobs that are not pending are purged.
|
25
|
+
# @param before [ActiveSupport::Duration,Numeric,Time] the maximum scheduled at. Default: 48 hours ago
|
26
|
+
# @param status [Array<String,Symbol>,String,Symbol] the statuses to purge. Default: All except +:pending+
|
27
|
+
# @return [Integer] the number of purged jobs
|
28
|
+
def cleanup_jobs(before: 48.hours.ago, status: nil)
|
29
|
+
destroy_before = case before
|
30
|
+
when ActiveSupport::Duration
|
31
|
+
before.ago
|
32
|
+
when Numeric
|
33
|
+
before.hours.ago
|
34
|
+
when Date, Time
|
35
|
+
before
|
36
|
+
else
|
37
|
+
raise ArgumentError, "Unsupported value for before: #{before.class}"
|
38
|
+
end
|
39
|
+
unless [Array, String, Symbol, NilClass].any?(&status.method(:is_a?))
|
40
|
+
raise ArgumentError, "Unsupported value for status: #{status.class}"
|
41
|
+
end
|
42
|
+
|
43
|
+
jobs = Exekutor::Job.all
|
44
|
+
unless before.nil?
|
45
|
+
jobs.where!(%{"scheduled_at"<?}, destroy_before)
|
46
|
+
end
|
47
|
+
if status
|
48
|
+
jobs.where! status: status
|
49
|
+
else
|
50
|
+
jobs = jobs.where.not(status: :p)
|
51
|
+
end
|
52
|
+
jobs.delete_all
|
53
|
+
end
|
54
|
+
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,373 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative "internal/configuration_builder"
|
4
|
+
|
5
|
+
module Exekutor
|
6
|
+
# Configuration for the Exekutor library
|
7
|
+
class Configuration
|
8
|
+
include Internal::ConfigurationBuilder
|
9
|
+
|
10
|
+
# @private
|
11
|
+
DEFAULT_BASE_RECORD_CLASS = "ActiveRecord::Base"
|
12
|
+
private_constant "DEFAULT_BASE_RECORD_CLASS"
|
13
|
+
|
14
|
+
# @!macro
|
15
|
+
# @!method $1
|
16
|
+
# Gets the default queue priority. Is used when enqueueing jobs that don't have a priority set.
|
17
|
+
# === Default value:
|
18
|
+
# 16,383
|
19
|
+
# @return [Integer]
|
20
|
+
# @!method $1=(value)
|
21
|
+
# Sets the default queue priority. Is used when enqueueing jobs that don't have a priority set. Should be
|
22
|
+
# between 1 and 32,767.
|
23
|
+
# @raise [Error] When the priority is nil or invalid
|
24
|
+
# @param value [Integer] the priority
|
25
|
+
# @return [self]
|
26
|
+
define_option :default_queue_priority, default: 16_383, required: true, type: Integer,
|
27
|
+
range: Exekutor::Queue::VALID_PRIORITIES
|
28
|
+
|
29
|
+
# @!macro
|
30
|
+
# @!method $1
|
31
|
+
# Gets the base class name for database records.
|
32
|
+
# === Default value:
|
33
|
+
# +"ActiveRecord::Base"+
|
34
|
+
# @return [String]
|
35
|
+
# @!method $1=(value)
|
36
|
+
# Sets the base class name for database records. The validity of this value will not be checked immediately.
|
37
|
+
# (Ie. When the specified class does not exist, an error will raised when a database record is used for the
|
38
|
+
# first time.)
|
39
|
+
# @raise [Error] When the name is blank
|
40
|
+
# @param value [String] the class name
|
41
|
+
# @return [self]
|
42
|
+
define_option :base_record_class_name, default: DEFAULT_BASE_RECORD_CLASS, required: true, type: String
|
43
|
+
|
44
|
+
# Gets the base class for database records. Is derived from the {#base_record_class_name} option.
|
45
|
+
# @raise [Error] when the class cannot be found
|
46
|
+
# @return [Class]
|
47
|
+
def base_record_class
|
48
|
+
const_get :base_record_class_name
|
49
|
+
rescue ::StandardError
|
50
|
+
# A nicer message for the default value
|
51
|
+
if base_record_class_name == DEFAULT_BASE_RECORD_CLASS
|
52
|
+
raise Error, "Cannot find ActiveRecord, did you install and load the gem?"
|
53
|
+
end
|
54
|
+
|
55
|
+
raise
|
56
|
+
end
|
57
|
+
|
58
|
+
# @!macro
|
59
|
+
# @!method $1
|
60
|
+
# Gets the unconverted JSON serializer value. This can be either a +String+, a +Symbol+, a +Proc+, or the
|
61
|
+
# serializer.
|
62
|
+
# === Default value:
|
63
|
+
# +JSON+
|
64
|
+
# @return [String,Symbol,Proc,Object]
|
65
|
+
# @!method $1=(value)
|
66
|
+
# Sets the JSON serializer. This can be either a +String+, a +Symbol+, a +Proc+, or the serializer. If a
|
67
|
+
# +String+, +Symbol+, or +Proc+ is given, the serializer will be loaded when it is needed for the first time.
|
68
|
+
# If the loaded class does not respond to +#dump+ and +#load+ an {Error} will be raised whenever the serializer
|
69
|
+
# is loaded. If the value is neither a +String+, +Symbol+, nor a +Proc+ and it does not respond to +#dump+ and
|
70
|
+
# +#load+, the error will be thrown immediately.
|
71
|
+
# @raise [Error] When the value is neither a +String+, +Symbol+, nor a +Proc+ and it does not respond to +#dump+
|
72
|
+
# and +#load+
|
73
|
+
# @param value [String,Symbol,Proc,Object] the serializer
|
74
|
+
# @return [self]
|
75
|
+
define_option :json_serializer, default: "::JSON", required: true do |value|
|
76
|
+
unless value.is_a?(String) || value.is_a?(Symbol) || value.respond_to?(:call) ||
|
77
|
+
(value.respond_to?(:dump) && value.respond_to?(:load))
|
78
|
+
raise Error, "#json_serializer must either be a String, a Proc, or respond to #dump and #load"
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
# Gets the JSON serializer. Is derived from the {#json_serializer} option.
|
83
|
+
# @raise [Error] when the class cannot be found, or does not respond to +#dump+ and +#load+
|
84
|
+
# @return [Object]
|
85
|
+
def load_json_serializer
|
86
|
+
raw_value = self.json_serializer
|
87
|
+
if defined?(@json_serializer_instance) && @json_serializer_instance[0] == raw_value
|
88
|
+
return @json_serializer_instance[1]
|
89
|
+
end
|
90
|
+
|
91
|
+
serializer = const_get :json_serializer
|
92
|
+
unless serializer.respond_to?(:dump) && serializer.respond_to?(:load)
|
93
|
+
serializer = serializer.call if serializer.respond_to?(:call)
|
94
|
+
unless serializer.respond_to?(:dump) && serializer.respond_to?(:load)
|
95
|
+
serializer = serializer.new if serializer.respond_to?(:new)
|
96
|
+
end
|
97
|
+
end
|
98
|
+
unless serializer.respond_to?(:dump) && serializer.respond_to?(:load)
|
99
|
+
raise Error, <<~MSG.squish
|
100
|
+
The configured serializer (#{serializer.class}) does not respond to #dump and #load
|
101
|
+
MSG
|
102
|
+
end
|
103
|
+
|
104
|
+
@json_serializer_instance = [raw_value, serializer]
|
105
|
+
serializer
|
106
|
+
end
|
107
|
+
|
108
|
+
# @!macro
|
109
|
+
# @!method $1
|
110
|
+
# Gets the logger.
|
111
|
+
# === Default value:
|
112
|
+
# Rails.active_job.logger
|
113
|
+
# @return [ActiveSupport::Logger]
|
114
|
+
# @!method $1=(value)
|
115
|
+
# Sets the logger.
|
116
|
+
# @param value [ActiveSupport::Logger] the logger
|
117
|
+
# @return [self]
|
118
|
+
define_option :logger, default: -> { Rails.active_job.logger }
|
119
|
+
|
120
|
+
# @!macro
|
121
|
+
# @!method $1?
|
122
|
+
# Whether the DB connection name should be set. Only affects the listener, unless started from the CLI.
|
123
|
+
# === Default value:
|
124
|
+
# false (true when started from the CLI)
|
125
|
+
# @return [Boolean, nil]
|
126
|
+
# @!method $1=(value)
|
127
|
+
# Sets whether the DB connection name should be set
|
128
|
+
# @param value [Boolean] whether to name should be set
|
129
|
+
# @return [self]
|
130
|
+
define_option :set_db_connection_name, type: [TrueClass, FalseClass], required: true
|
131
|
+
|
132
|
+
def set_db_connection_name?
|
133
|
+
if set_db_connection_name.nil?
|
134
|
+
false
|
135
|
+
else
|
136
|
+
set_db_connection_name
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
140
|
+
# @!macro
|
141
|
+
# @!method $1?
|
142
|
+
# Whether the worker should use LISTEN/NOTIFY to listen for jobs.
|
143
|
+
# === Default value:
|
144
|
+
# true
|
145
|
+
# @return [Boolean, nil]
|
146
|
+
# @!method $1=(value)
|
147
|
+
# Sets whether the worker should use LISTEN/NOTIFY to listen for jobs
|
148
|
+
# @param value [Boolean] whether to enable the listener
|
149
|
+
# @return [self]
|
150
|
+
define_option :enable_listener, reader: :enable_listener?, default: true, type: [TrueClass, FalseClass],
|
151
|
+
required: true
|
152
|
+
|
153
|
+
# @!macro
|
154
|
+
# @!method $1?
|
155
|
+
# Whether the worker should delete jobs after completion.
|
156
|
+
# === Default value:
|
157
|
+
# false
|
158
|
+
# @return [Boolean]
|
159
|
+
# @!method $1=(value)
|
160
|
+
# Sets whether the worker should delete jobs after completion
|
161
|
+
# @param value [Boolean] whether to delete completed jobs
|
162
|
+
# @return [self]
|
163
|
+
define_option :delete_completed_jobs, reader: :delete_completed_jobs?, required: true,
|
164
|
+
type: [TrueClass, FalseClass], default: false
|
165
|
+
|
166
|
+
# @!macro
|
167
|
+
# @!method $1?
|
168
|
+
# Whether the worker should delete discarded jobs.
|
169
|
+
# === Default value:
|
170
|
+
# false
|
171
|
+
# @return [Boolean]
|
172
|
+
# @!method $1=(value)
|
173
|
+
# Sets whether the worker should delete discarded jobs
|
174
|
+
# @param value [Boolean] whether to delete discarded jobs
|
175
|
+
# @return [self]
|
176
|
+
define_option :delete_discarded_jobs, reader: :delete_discarded_jobs?, required: true,
|
177
|
+
type: [TrueClass, FalseClass], default: false
|
178
|
+
|
179
|
+
# @!macro
|
180
|
+
# @!method $1?
|
181
|
+
# Whether the worker should delete jobs after they failed to execute.
|
182
|
+
# === Default value:
|
183
|
+
# false
|
184
|
+
# @return [Boolean]
|
185
|
+
# @!method $1=(value)
|
186
|
+
# Sets whether the worker should delete jobs after they failed to execute
|
187
|
+
# @param value [Boolean] whether to delete failed jobs
|
188
|
+
# @return [self]
|
189
|
+
define_option :delete_failed_jobs, reader: :delete_failed_jobs?, required: true,
|
190
|
+
type: [TrueClass, FalseClass], default: false
|
191
|
+
|
192
|
+
# @!macro
|
193
|
+
# @!method $1
|
194
|
+
# The polling interval in seconds. When set, the worker will poll the database with this interval to check for
|
195
|
+
# any pending jobs that a listener might have missed (if enabled).
|
196
|
+
# === Default value:
|
197
|
+
# 60
|
198
|
+
# @return [Integer]
|
199
|
+
# @!method $1=(value)
|
200
|
+
# Sets the polling interval in seconds. Set to +nil+ to disable polling. If the listener is disabled, this value
|
201
|
+
# should be reasonably low so jobs don't have to wait in the queue too long; if it is enabled, this value can
|
202
|
+
# be reasonably high.
|
203
|
+
# @param value [Integer] the interval
|
204
|
+
# @return [self]
|
205
|
+
define_option :polling_interval, default: 60, type: [Integer, nil], range: 1...(1.day.to_i)
|
206
|
+
|
207
|
+
# @!macro
|
208
|
+
# @!method $1
|
209
|
+
# The polling jitter, used to adjust the polling interval slightly so multiple workers will not query the
|
210
|
+
# database at the same time.
|
211
|
+
# === Default value:
|
212
|
+
# 0.1
|
213
|
+
# @return [Float]
|
214
|
+
# @!method $1=(value)
|
215
|
+
# Sets the polling jitter, which is used to slightly adjust the polling interval. Should be between 0 and 0.5.
|
216
|
+
# A value of 0.1 means the polling interval can vary by 10%. If the interval is set to 60 seconds and the jitter
|
217
|
+
# is set to 0.1, the interval can range from 57 to 63 seconds. A value of 0 disables this feature.
|
218
|
+
# @param value [Float] the jitter
|
219
|
+
# @return [self]
|
220
|
+
define_option :polling_jitter, default: 0.1, type: [Float, Integer], range: 0..0.5
|
221
|
+
|
222
|
+
# @!macro
|
223
|
+
# @!method $1
|
224
|
+
# The minimum number of execution threads that should be active.
|
225
|
+
# === Default value:
|
226
|
+
# 1
|
227
|
+
# @return [Integer]
|
228
|
+
# @!method $1=(value)
|
229
|
+
# Sets the minimum number of execution threads that should be active
|
230
|
+
# @param value [Integer] the number of threads
|
231
|
+
# @return [self]
|
232
|
+
define_option :min_execution_threads, default: 1, type: Integer, range: 1...999
|
233
|
+
|
234
|
+
# @!macro
|
235
|
+
# @!method $1
|
236
|
+
# The maximum number of execution threads that may be active.
|
237
|
+
# === Default value:
|
238
|
+
# Active record pool size minus 1, with a minimum of 1
|
239
|
+
# @return [Integer]
|
240
|
+
# @!method $1=(value)
|
241
|
+
# Sets the maximum number of execution threads that may be active. Be aware that if you set this to a value
|
242
|
+
# greater than +connection_db_config.pool+, workers may have to wait for database connections to become
|
243
|
+
# available because all connections are occupied by other threads. This may result in an
|
244
|
+
# +ActiveRecord::ConnectionTimeoutError+ if the thread has to wait too long.
|
245
|
+
# @param value [Integer] the number of threads
|
246
|
+
# @return [self]
|
247
|
+
define_option :max_execution_threads,
|
248
|
+
default: -> { (Internal::BaseRecord.connection_db_config.pool.to_i - 1).clamp(1, 999) },
|
249
|
+
type: Integer, range: 1...999
|
250
|
+
|
251
|
+
# @!macro
|
252
|
+
# @!method $1
|
253
|
+
# The maximum number of seconds a thread may be idle before being stopped.
|
254
|
+
# === Default value:
|
255
|
+
# 60
|
256
|
+
# @return [Integer]
|
257
|
+
# @!method $1=(value)
|
258
|
+
# Sets the maximum number of seconds a thread may be idle before being stopped
|
259
|
+
# @param value [Integer] the number of threads
|
260
|
+
# @return [self]
|
261
|
+
define_option :max_execution_thread_idletime, default: 60, type: Integer, range: 1..(1.day.to_i)
|
262
|
+
|
263
|
+
# @!macro
|
264
|
+
# @!method $1?
|
265
|
+
# The rack handler for the status server
|
266
|
+
# === Default value:
|
267
|
+
# webrick
|
268
|
+
# @return [String]
|
269
|
+
# @!method $1=(value)
|
270
|
+
# Sets the rack handler for the status server. The handler should respond to +#shutdown+ or +#stop+.
|
271
|
+
# @param value [String] the name of the handler
|
272
|
+
# @return [self]
|
273
|
+
define_option :status_server_handler, default: "webrick", type: String
|
274
|
+
|
275
|
+
# @!macro
|
276
|
+
# @!method $1?
|
277
|
+
# The heartbeat timeout for the `/live` endpoint of the status server, in minutes. If the heartbeat of a worker
|
278
|
+
# is older than this timeout, the status server will respond with a 503 status indicating the service is
|
279
|
+
# down.
|
280
|
+
# === Default value:
|
281
|
+
# 30
|
282
|
+
# @return [Integer]
|
283
|
+
# @!method $1=(value)
|
284
|
+
# Sets the heartbeat timeout for the `/live` endpoint of the status server, in minutes. Must be between 2
|
285
|
+
# and 1440 (24 hours).
|
286
|
+
# @param value [Integer] The timeout in minutes
|
287
|
+
# @return [self]
|
288
|
+
define_option :healthcheck_timeout, default: 30, type: Integer, range: 2..1440
|
289
|
+
|
290
|
+
# @!macro
|
291
|
+
# @!method $1?
|
292
|
+
# Whether to suppress STDOUT messages
|
293
|
+
# === Default value:
|
294
|
+
# false
|
295
|
+
# @return [Boolean]
|
296
|
+
# @!method $1=(value)
|
297
|
+
# Sets whether the STDOUT messages should be printed
|
298
|
+
# @param value [Boolean] whether to suppress STDOUT messages
|
299
|
+
# @return [self]
|
300
|
+
define_option :quiet, reader: :quiet?, type: [TrueClass, FalseClass], required: true, default: false
|
301
|
+
|
302
|
+
# Gets the options for a worker
|
303
|
+
# @return [Hash] the worker configuration
|
304
|
+
def worker_options
|
305
|
+
{
|
306
|
+
min_threads: min_execution_threads,
|
307
|
+
max_threads: max_execution_threads,
|
308
|
+
max_thread_idletime: max_execution_thread_idletime,
|
309
|
+
}.tap do |opts|
|
310
|
+
opts[:set_db_connection_name] = set_db_connection_name? unless set_db_connection_name.nil?
|
311
|
+
%i[enable_listener delete_completed_jobs delete_discarded_jobs delete_failed_jobs].each do |option|
|
312
|
+
opts[option] = send(:"#{option}?") ? true : false
|
313
|
+
end
|
314
|
+
%i[polling_interval polling_jitter status_server_handler healthcheck_timeout].each do |option|
|
315
|
+
opts[option] = send(option)
|
316
|
+
end
|
317
|
+
end
|
318
|
+
end
|
319
|
+
|
320
|
+
private
|
321
|
+
|
322
|
+
def const_get(option_name)
|
323
|
+
class_name = send(option_name)
|
324
|
+
case class_name
|
325
|
+
when String, Symbol
|
326
|
+
begin
|
327
|
+
class_name = if class_name.is_a? Symbol
|
328
|
+
class_name.to_s.camelize.prepend("::")
|
329
|
+
elsif class_name.start_with? "::"
|
330
|
+
class_name
|
331
|
+
else
|
332
|
+
class_name.dup.prepend("::")
|
333
|
+
end
|
334
|
+
|
335
|
+
Object.const_get class_name
|
336
|
+
rescue NameError, LoadError
|
337
|
+
raise Error, <<~MSG.squish
|
338
|
+
Cannot convert ##{option_name} (#{class_name.inspect}) to a constant. Have you made a typo?
|
339
|
+
MSG
|
340
|
+
end
|
341
|
+
else
|
342
|
+
class_name
|
343
|
+
end
|
344
|
+
end
|
345
|
+
|
346
|
+
# Raised when configuring an invalid option or value
|
347
|
+
class Error < Exekutor::Error; end
|
348
|
+
|
349
|
+
protected
|
350
|
+
|
351
|
+
def error_class
|
352
|
+
Error
|
353
|
+
end
|
354
|
+
end
|
355
|
+
|
356
|
+
def self.config
|
357
|
+
@config ||= Exekutor::Configuration.new
|
358
|
+
end
|
359
|
+
|
360
|
+
def self.configure(opts = nil, &block)
|
361
|
+
raise ArgumentError, "opts must be a Hash" unless opts.nil? || opts.is_a?(Hash)
|
362
|
+
raise ArgumentError, "Either opts or a block must be given" unless opts.present? || block_given?
|
363
|
+
|
364
|
+
config.set(**opts) if opts
|
365
|
+
return unless block_given?
|
366
|
+
|
367
|
+
if block.arity == 1
|
368
|
+
block.call config
|
369
|
+
else
|
370
|
+
instance_eval(&block)
|
371
|
+
end
|
372
|
+
end
|
373
|
+
end
|