rocketjob 6.3.2 → 7.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +38 -316
- data/bin/rocketjob_batch_perf +1 -1
- data/lib/rocket_job/batch/categories.rb +25 -25
- data/lib/rocket_job/batch/io.rb +21 -15
- data/lib/rocket_job/batch/logger.rb +2 -2
- data/lib/rocket_job/batch/model.rb +2 -2
- data/lib/rocket_job/batch/performance.rb +1 -1
- data/lib/rocket_job/batch/statistics.rb +9 -2
- data/lib/rocket_job/batch/throttle.rb +7 -2
- data/lib/rocket_job/batch/throttle_running_workers.rb +5 -1
- data/lib/rocket_job/batch/throttle_windows.rb +4 -2
- data/lib/rocket_job/batch/worker.rb +51 -11
- data/lib/rocket_job/config.rb +6 -1
- data/lib/rocket_job/dirmon_entry.rb +6 -4
- data/lib/rocket_job/event.rb +84 -4
- data/lib/rocket_job/extensions/mongo/logging.rb +1 -0
- data/lib/rocket_job/extensions/mongoid/clients/options.rb +3 -3
- data/lib/rocket_job/extensions/mongoid/contextual/mongo.rb +3 -4
- data/lib/rocket_job/extensions/mongoid/factory.rb +2 -2
- data/lib/rocket_job/extensions/psych/yaml_tree.rb +8 -4
- data/lib/rocket_job/extensions/rocket_job_adapter.rb +1 -1
- data/lib/rocket_job/jobs/active_job.rb +1 -1
- data/lib/rocket_job/jobs/dirmon_job.rb +3 -1
- data/lib/rocket_job/jobs/on_demand_batch_job.rb +1 -1
- data/lib/rocket_job/jobs/upload_file_job.rb +8 -5
- data/lib/rocket_job/lookup_collection.rb +2 -2
- data/lib/rocket_job/performance.rb +1 -1
- data/lib/rocket_job/plugins/cron.rb +6 -3
- data/lib/rocket_job/plugins/document.rb +0 -2
- data/lib/rocket_job/plugins/job/logger.rb +2 -2
- data/lib/rocket_job/plugins/job/model.rb +1 -1
- data/lib/rocket_job/plugins/job/persistence.rb +2 -2
- data/lib/rocket_job/plugins/job/throttle.rb +14 -2
- data/lib/rocket_job/plugins/job/throttle_running_jobs.rb +4 -1
- data/lib/rocket_job/plugins/job/worker.rb +5 -2
- data/lib/rocket_job/plugins/retry.rb +1 -1
- data/lib/rocket_job/plugins/throttle_dependent_jobs.rb +7 -2
- data/lib/rocket_job/plugins/transaction.rb +2 -2
- data/lib/rocket_job/sliced/input.rb +17 -12
- data/lib/rocket_job/sliced/slice.rb +3 -3
- data/lib/rocket_job/sliced/slices.rb +3 -6
- data/lib/rocket_job/supervisor.rb +2 -2
- data/lib/rocket_job/thread_worker.rb +3 -3
- data/lib/rocket_job/throttle_definition.rb +19 -2
- data/lib/rocket_job/throttle_definitions.rb +14 -9
- data/lib/rocket_job/version.rb +1 -1
- data/lib/rocket_job/worker.rb +77 -28
- data/lib/rocketjob.rb +1 -2
- metadata +33 -27
- data/lib/rocket_job/ractor_worker.rb +0 -42
|
@@ -36,7 +36,11 @@ module RocketJob
|
|
|
36
36
|
|
|
37
37
|
validates :throttle_running_workers, numericality: {greater_than_or_equal_to: 0}, allow_nil: true
|
|
38
38
|
|
|
39
|
-
define_batch_throttle :throttle_running_workers_exceeded?,
|
|
39
|
+
define_batch_throttle :throttle_running_workers_exceeded?,
|
|
40
|
+
filter: :throttle_filter_id,
|
|
41
|
+
description: lambda { |job, *|
|
|
42
|
+
"Throttled: maximum of #{job.throttle_running_workers} running workers reached"
|
|
43
|
+
}
|
|
40
44
|
end
|
|
41
45
|
|
|
42
46
|
private
|
|
@@ -43,7 +43,9 @@ module RocketJob
|
|
|
43
43
|
# Duration in seconds of the secondary window.
|
|
44
44
|
field :secondary_duration, type: Integer, class_attribute: true, user_editable: true, copy_on_restart: true
|
|
45
45
|
|
|
46
|
-
define_batch_throttle :throttle_windows_exceeded?,
|
|
46
|
+
define_batch_throttle :throttle_windows_exceeded?,
|
|
47
|
+
filter: :throttle_filter_id,
|
|
48
|
+
description: "Throttled: outside of its processing window"
|
|
47
49
|
|
|
48
50
|
validates_each :primary_schedule, :secondary_schedule do |record, attr, value|
|
|
49
51
|
record.errors.add(attr, "Invalid #{attr}: #{value.inspect}") if value && !Fugit::Cron.new(value)
|
|
@@ -62,7 +64,7 @@ module RocketJob
|
|
|
62
64
|
|
|
63
65
|
def throttle_outside_window?(schedule, duration)
|
|
64
66
|
cron = Fugit::Cron.new(schedule)
|
|
65
|
-
time = Time.now.
|
|
67
|
+
time = Time.now.getutc + 1
|
|
66
68
|
# Add 1 second since right now could be the very beginning of the processing window.
|
|
67
69
|
previous_time = cron.previous_time(time).to_utc_time
|
|
68
70
|
previous_time + duration < time
|
|
@@ -69,7 +69,7 @@ module RocketJob
|
|
|
69
69
|
# Note: The slice will be removed from processing when this method completes
|
|
70
70
|
#
|
|
71
71
|
# @deprecated Please open a ticket if you need this behavior.
|
|
72
|
-
def work_first_slice(&
|
|
72
|
+
def work_first_slice(&)
|
|
73
73
|
raise "#work_first_slice can only be called from within before_batch callbacks" unless sub_state == :before
|
|
74
74
|
|
|
75
75
|
# TODO: Make these settings configurable
|
|
@@ -89,7 +89,7 @@ module RocketJob
|
|
|
89
89
|
|
|
90
90
|
# TODO: Persist that the first slice is being processed by this worker
|
|
91
91
|
slice.start
|
|
92
|
-
rocket_job_process_slice(slice, &
|
|
92
|
+
rocket_job_process_slice(slice, &)
|
|
93
93
|
end
|
|
94
94
|
|
|
95
95
|
# Returns [Array<ActiveWorker>] All workers actively working on this job
|
|
@@ -111,15 +111,52 @@ module RocketJob
|
|
|
111
111
|
end
|
|
112
112
|
|
|
113
113
|
def rocket_job_batch_throttled?(slice, worker)
|
|
114
|
-
|
|
115
|
-
|
|
114
|
+
throttle = self.class.rocket_job_batch_throttles.matching_throttle(self, slice)
|
|
115
|
+
unless throttle
|
|
116
|
+
rocket_job_clear_throttled
|
|
117
|
+
return false
|
|
118
|
+
end
|
|
116
119
|
|
|
117
120
|
# Restore retrieved slice so that other workers can process it later.
|
|
118
121
|
slice.set(worker_name: nil, state: :queued, started_at: nil)
|
|
119
|
-
worker.add_to_current_filter(
|
|
122
|
+
worker.add_to_current_filter(throttle.extract_filter(self, slice))
|
|
123
|
+
rocket_job_set_throttled(throttle.extract_description(self, slice))
|
|
120
124
|
true
|
|
121
125
|
end
|
|
122
126
|
|
|
127
|
+
# Record why this batch job's slices are being throttled, for visibility in
|
|
128
|
+
# Mission Control.
|
|
129
|
+
#
|
|
130
|
+
# Unlike a simple job, a running batch job document is shared by every worker
|
|
131
|
+
# and is deliberately not written on each poll (see Worker#find_and_assign_job).
|
|
132
|
+
# The reason is therefore persisted with a guarded conditional update so the
|
|
133
|
+
# shared document is only written when the reason actually changes, not on
|
|
134
|
+
# every throttled poll.
|
|
135
|
+
def rocket_job_set_throttled(reason)
|
|
136
|
+
return if throttled_by == reason
|
|
137
|
+
|
|
138
|
+
self.class.with(write: {w: 1}) do |query|
|
|
139
|
+
query.where(id: id, state: :running, :throttled_by.ne => reason).
|
|
140
|
+
update_all("$set" => {throttled_by: reason, throttled_at: Time.now})
|
|
141
|
+
end
|
|
142
|
+
self.throttled_by = reason
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
# Clear the throttle reason once slices are flowing again.
|
|
146
|
+
#
|
|
147
|
+
# Gated on the in-memory value so a worker that never throttled this job
|
|
148
|
+
# issues no database round trip per slice; only a worker that previously set
|
|
149
|
+
# a reason clears it, with a guarded update to avoid a redundant write.
|
|
150
|
+
def rocket_job_clear_throttled
|
|
151
|
+
return if throttled_by.nil?
|
|
152
|
+
|
|
153
|
+
self.class.with(write: {w: 1}) do |query|
|
|
154
|
+
query.where(id: id, :throttled_by.ne => nil).
|
|
155
|
+
update_all("$unset" => {throttled_by: "", throttled_at: ""})
|
|
156
|
+
end
|
|
157
|
+
self.throttled_by = nil
|
|
158
|
+
end
|
|
159
|
+
|
|
123
160
|
# Process a single slice from Mongo
|
|
124
161
|
# Once the slice has been successfully processed it will be removed from the input collection
|
|
125
162
|
# Returns [Integer] the number of records successfully processed
|
|
@@ -151,7 +188,7 @@ module RocketJob
|
|
|
151
188
|
if slice.processing_record_number.to_i > 1
|
|
152
189
|
append = true
|
|
153
190
|
logger.info("Resuming previously incomplete slice from record number #{slice.processing_record_number}")
|
|
154
|
-
slice.records[slice.processing_record_number - 1
|
|
191
|
+
slice.records[(slice.processing_record_number - 1)..]
|
|
155
192
|
else
|
|
156
193
|
# Reprocess all records in this slice.
|
|
157
194
|
slice.processing_record_number = 0
|
|
@@ -162,8 +199,11 @@ module RocketJob
|
|
|
162
199
|
RocketJob::Sliced::Writer::Output.collect(self, input_slice: slice, append: append) do |writer|
|
|
163
200
|
records.each do |record|
|
|
164
201
|
slice.processing_record_number += 1
|
|
165
|
-
|
|
166
|
-
|
|
202
|
+
# `current_record_number` reads Mongoid fields, so compute it once per
|
|
203
|
+
# record and reuse it for both the log tag and `#perform`.
|
|
204
|
+
record_number = slice.current_record_number
|
|
205
|
+
SemanticLogger.named_tagged(record: record_number) do
|
|
206
|
+
writer << rocket_job_batch_perform(slice, record, record_number, &block)
|
|
167
207
|
count += 1
|
|
168
208
|
end
|
|
169
209
|
end
|
|
@@ -172,8 +212,8 @@ module RocketJob
|
|
|
172
212
|
end
|
|
173
213
|
|
|
174
214
|
# Perform a single record within the current slice.
|
|
175
|
-
def rocket_job_batch_perform(slice, record)
|
|
176
|
-
@rocket_job_record_number =
|
|
215
|
+
def rocket_job_batch_perform(slice, record, record_number = slice.current_record_number)
|
|
216
|
+
@rocket_job_record_number = record_number
|
|
177
217
|
|
|
178
218
|
return block_given? ? yield(record) : perform(record) if _perform_callbacks.empty?
|
|
179
219
|
|
|
@@ -248,7 +288,7 @@ module RocketJob
|
|
|
248
288
|
unless new_record?
|
|
249
289
|
# Fail job iff no other worker has already finished it
|
|
250
290
|
# Must set write concern to at least 1 since we need the nModified back
|
|
251
|
-
result
|
|
291
|
+
result = self.class.with(write: {w: 1}) do |query|
|
|
252
292
|
query.
|
|
253
293
|
where(id: id, state: :running, sub_state: :processing).
|
|
254
294
|
update({"$set" => {state: "failed", worker_name: worker_name}})
|
data/lib/rocket_job/config.rb
CHANGED
|
@@ -81,6 +81,11 @@ module RocketJob
|
|
|
81
81
|
logger.debug "Reading Mongo configuration from: #{config_file}"
|
|
82
82
|
::Mongoid.load!(config_file, environment)
|
|
83
83
|
|
|
84
|
+
# Mongoid 9 requires `Time.zone` to be set in order to mongoize Date and
|
|
85
|
+
# Time values. When running outside of Rails it may be unset, so default
|
|
86
|
+
# it here without overriding an explicit application setting.
|
|
87
|
+
::Time.zone_default ||= ::ActiveSupport::TimeZone["UTC"]
|
|
88
|
+
|
|
84
89
|
config_file =
|
|
85
90
|
if encryption_file_name
|
|
86
91
|
Pathname.new(encryption_file_name)
|
|
@@ -100,7 +105,7 @@ module RocketJob
|
|
|
100
105
|
def self.filter
|
|
101
106
|
raise(ArgumentError, "Cannot supply both an include_filter and an exclude_filter") if include_filter && exclude_filter
|
|
102
107
|
|
|
103
|
-
filter
|
|
108
|
+
filter = where_filter
|
|
104
109
|
(filter ||= {})["_type"] = include_filter if include_filter
|
|
105
110
|
(filter ||= {})["_type"] = {"$not" => exclude_filter} if exclude_filter
|
|
106
111
|
filter&.dup
|
|
@@ -286,14 +286,16 @@ module RocketJob
|
|
|
286
286
|
klass = job_class
|
|
287
287
|
return unless klass
|
|
288
288
|
|
|
289
|
-
|
|
290
|
-
|
|
289
|
+
# Mongoid 9 returns Hash field keys as Strings, earlier versions as Symbols.
|
|
290
|
+
properties.each_pair do |raw_key, value|
|
|
291
|
+
k = raw_key.to_sym
|
|
292
|
+
next if klass.public_method_defined?(:"#{k}=")
|
|
291
293
|
|
|
292
294
|
if %i[output_categories input_categories].include?(k)
|
|
293
295
|
category_class = k == :input_categories ? RocketJob::Category::Input : RocketJob::Category::Output
|
|
294
|
-
|
|
296
|
+
value.each do |category|
|
|
295
297
|
category.each_pair do |key, _value|
|
|
296
|
-
next if category_class.public_method_defined?("#{key}="
|
|
298
|
+
next if category_class.public_method_defined?(:"#{key}=")
|
|
297
299
|
|
|
298
300
|
errors.add(
|
|
299
301
|
:properties,
|
data/lib/rocket_job/event.rb
CHANGED
|
@@ -12,6 +12,11 @@ module RocketJob
|
|
|
12
12
|
|
|
13
13
|
ALL_EVENTS = "*".freeze
|
|
14
14
|
|
|
15
|
+
# MongoDB OperationFailure codes that are safe to ignore when concurrently
|
|
16
|
+
# creating the polling collection and its TTL index:
|
|
17
|
+
# 48 NamespaceExists, 85 IndexOptionsConflict, 86 IndexKeySpecsConflict.
|
|
18
|
+
IGNORABLE_CREATE_CODES = [48, 85, 86].freeze
|
|
19
|
+
|
|
15
20
|
# Capped collection long polling interval.
|
|
16
21
|
class_attribute :long_poll_seconds, instance_accessor: false
|
|
17
22
|
self.long_poll_seconds = 300
|
|
@@ -23,6 +28,28 @@ module RocketJob
|
|
|
23
28
|
class_attribute :capped_collection_size, instance_accessor: false
|
|
24
29
|
self.capped_collection_size = 128 * 1024 * 1024
|
|
25
30
|
|
|
31
|
+
# Event listener strategy:
|
|
32
|
+
# :capped - Tail a capped collection with a tailable cursor (default).
|
|
33
|
+
# Lowest latency, but requires a data store that supports capped
|
|
34
|
+
# collections and tailable cursors (i.e. a real MongoDB server).
|
|
35
|
+
# :polling - Poll a regular collection for new events. Slightly higher latency
|
|
36
|
+
# (bounded by `poll_interval`), but works on any MongoDB-compatible
|
|
37
|
+
# store, including AWS DocumentDB which has no capped collections.
|
|
38
|
+
class_attribute :listener_strategy, instance_accessor: false
|
|
39
|
+
self.listener_strategy = :capped
|
|
40
|
+
|
|
41
|
+
# Polling strategy: seconds to wait between polls for new events.
|
|
42
|
+
class_attribute :poll_interval, instance_accessor: false
|
|
43
|
+
self.poll_interval = 1
|
|
44
|
+
|
|
45
|
+
# Polling strategy: seconds an event is retained before a TTL index removes it.
|
|
46
|
+
# Must comfortably exceed the longest expected listener downtime so that a
|
|
47
|
+
# restarting server can still recover events published while it was away.
|
|
48
|
+
#
|
|
49
|
+
# Default: 1 hour.
|
|
50
|
+
class_attribute :event_retention_seconds, instance_accessor: false
|
|
51
|
+
self.event_retention_seconds = 60 * 60
|
|
52
|
+
|
|
26
53
|
# Mandatory Event Name
|
|
27
54
|
# Examples:
|
|
28
55
|
# '/rocket_job/config'
|
|
@@ -81,14 +108,23 @@ module RocketJob
|
|
|
81
108
|
@subscribers.each_value { |v| v.delete_if { |i| i.object_id == handle } }
|
|
82
109
|
end
|
|
83
110
|
|
|
84
|
-
# Indefinitely
|
|
111
|
+
# Indefinitely watch for new events, dispatching each to its subscribers.
|
|
85
112
|
# time: the start time from which to start looking for new events.
|
|
86
113
|
def self.listener(time: @load_time)
|
|
87
114
|
Thread.current.name = "rocketjob event"
|
|
88
|
-
create_capped_collection
|
|
89
115
|
|
|
90
|
-
|
|
91
|
-
|
|
116
|
+
case listener_strategy
|
|
117
|
+
when :capped
|
|
118
|
+
create_capped_collection
|
|
119
|
+
logger.info("Event listener started (capped collection)")
|
|
120
|
+
tail_capped_collection(time) { |event| process_event(event) }
|
|
121
|
+
when :polling
|
|
122
|
+
create_polling_collection
|
|
123
|
+
logger.info("Event listener started (polling, interval: #{poll_interval}s)")
|
|
124
|
+
poll_collection(time) { |event| process_event(event) }
|
|
125
|
+
else
|
|
126
|
+
raise(ArgumentError, "Unknown RocketJob::Event.listener_strategy: #{listener_strategy.inspect}")
|
|
127
|
+
end
|
|
92
128
|
rescue Exception => e
|
|
93
129
|
logger.error("#listener Event listener is terminating due to unhandled exception", e)
|
|
94
130
|
raise(e)
|
|
@@ -134,6 +170,50 @@ module RocketJob
|
|
|
134
170
|
retry
|
|
135
171
|
end
|
|
136
172
|
|
|
173
|
+
# Create the regular (non-capped) collection used by the polling strategy,
|
|
174
|
+
# along with a TTL index that expires old events.
|
|
175
|
+
#
|
|
176
|
+
# Safe to call repeatedly: it never drops data, and tolerates the collection
|
|
177
|
+
# and index already existing.
|
|
178
|
+
def self.create_polling_collection
|
|
179
|
+
collection.client[collection_name].create unless collection_exists?
|
|
180
|
+
collection.indexes.create_one({created_at: 1}, expire_after: event_retention_seconds)
|
|
181
|
+
rescue Mongo::Error::OperationFailure => e
|
|
182
|
+
# Ignore "collection already exists" and "index already exists" races between
|
|
183
|
+
# multiple servers starting at once. Anything else is a real error.
|
|
184
|
+
raise(e) unless IGNORABLE_CREATE_CODES.include?(e.code)
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
# Indefinitely poll a regular collection for new events.
|
|
188
|
+
# time: the start time from which to start looking for new events.
|
|
189
|
+
#
|
|
190
|
+
# After the first poll the `_id` of the last event seen is used as the
|
|
191
|
+
# watermark, which is monotonic and unique, so events sharing a `created_at`
|
|
192
|
+
# timestamp are never skipped.
|
|
193
|
+
def self.poll_collection(time, &block)
|
|
194
|
+
last_id = nil
|
|
195
|
+
loop do
|
|
196
|
+
last_id = poll_once(time, last_id, &block)
|
|
197
|
+
sleep(poll_interval)
|
|
198
|
+
end
|
|
199
|
+
rescue Mongo::Error::SocketError, Mongo::Error::SocketTimeoutError, Mongo::Error::OperationFailure, Timeout::Error => e
|
|
200
|
+
logger.info("Polling failed, retrying: #{e.class.name} #{e.message}")
|
|
201
|
+
sleep(poll_interval)
|
|
202
|
+
retry
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
# Perform a single polling pass, yielding every event newer than the watermark
|
|
206
|
+
# and returning the new watermark (the `_id` of the last event seen, or the
|
|
207
|
+
# supplied `last_id` when no new events were found).
|
|
208
|
+
def self.poll_once(time, last_id = nil)
|
|
209
|
+
filter = last_id ? {_id: {"$gt" => last_id}} : {created_at: {"$gt" => time}}
|
|
210
|
+
collection.find(filter).sort(_id: 1).each do |doc|
|
|
211
|
+
last_id = doc["_id"]
|
|
212
|
+
yield(Mongoid::Factory.from_db(Event, doc))
|
|
213
|
+
end
|
|
214
|
+
last_id
|
|
215
|
+
end
|
|
216
|
+
|
|
137
217
|
# Process a new event, calling registered subscribers.
|
|
138
218
|
def self.process_event(event)
|
|
139
219
|
logger.info("Event Received", event.attributes)
|
|
@@ -11,7 +11,7 @@ module RocketJob
|
|
|
11
11
|
end
|
|
12
12
|
|
|
13
13
|
def collection(parent = nil)
|
|
14
|
-
@collection_name ? mongo_client[@collection_name] : super
|
|
14
|
+
@collection_name ? mongo_client[@collection_name] : super
|
|
15
15
|
end
|
|
16
16
|
|
|
17
17
|
def collection_name
|
|
@@ -31,5 +31,5 @@ module RocketJob
|
|
|
31
31
|
end
|
|
32
32
|
end
|
|
33
33
|
|
|
34
|
-
|
|
35
|
-
|
|
34
|
+
Mongoid::Criteria.include(RocketJob::MongoidClients::Options)
|
|
35
|
+
Mongoid::Document.include(RocketJob::MongoidClients::Options)
|
|
@@ -1,15 +1,14 @@
|
|
|
1
|
-
::Mongoid::Contextual::Mongo
|
|
2
1
|
module Mongoid
|
|
3
2
|
module Contextual
|
|
4
3
|
class Mongo
|
|
5
4
|
def initialize(criteria)
|
|
6
5
|
@criteria = criteria
|
|
7
6
|
@klass = criteria.klass
|
|
8
|
-
|
|
9
|
-
#
|
|
7
|
+
# Only line changed from the Mongoid implementation is here: fetch the
|
|
8
|
+
# collection from the criteria so a custom collection_name is honored
|
|
9
|
+
# rather than always using @klass.collection.
|
|
10
10
|
# @collection = @klass.collection
|
|
11
11
|
@collection = criteria.collection
|
|
12
|
-
|
|
13
12
|
criteria.send(:merge_type_selection)
|
|
14
13
|
@view = collection.find(criteria.selector, session: _session)
|
|
15
14
|
apply_options
|
|
@@ -4,11 +4,11 @@ module RocketJob
|
|
|
4
4
|
# Don't convert to Mongoid::Factory since it conflicts with Mongoid use.
|
|
5
5
|
module MongoidFactory
|
|
6
6
|
def from_db(klass, attributes = nil, criteria = nil, selected_fields = nil)
|
|
7
|
-
obj = super
|
|
7
|
+
obj = super
|
|
8
8
|
obj.collection_name = criteria.collection_name if criteria
|
|
9
9
|
obj
|
|
10
10
|
end
|
|
11
11
|
end
|
|
12
12
|
end
|
|
13
13
|
|
|
14
|
-
|
|
14
|
+
Mongoid::Factory.extend(RocketJob::MongoidFactory)
|
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
require "psych/visitors/yaml_tree"
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
3
|
+
module Psych
|
|
4
|
+
module Visitors
|
|
5
|
+
class YAMLTree
|
|
6
|
+
# Serialize IOStream path as a string
|
|
7
|
+
def visit_IOStreams_Path(o)
|
|
8
|
+
visit_String(o.to_s)
|
|
9
|
+
end
|
|
10
|
+
end
|
|
7
11
|
end
|
|
8
12
|
end
|
|
@@ -4,7 +4,7 @@ module ActiveJob
|
|
|
4
4
|
#
|
|
5
5
|
# Ruby's missing batch system.
|
|
6
6
|
#
|
|
7
|
-
# Read more about Rocket Job {here}[
|
|
7
|
+
# Read more about Rocket Job {here}[https://rocketjob.reidmorrison.com].
|
|
8
8
|
#
|
|
9
9
|
# To use Rocket Job set the queue_adapter config to +:rocket_job+.
|
|
10
10
|
#
|
|
@@ -85,7 +85,9 @@ module RocketJob
|
|
|
85
85
|
file_names[key] = size if size
|
|
86
86
|
end
|
|
87
87
|
rescue StandardError => e
|
|
88
|
-
logger.error(
|
|
88
|
+
logger.error(
|
|
89
|
+
"Dirmon Entry: #{dirmon_entry.id} failed. Moved to `failed` state to prevent processing again without manual intervention.", e
|
|
90
|
+
)
|
|
89
91
|
dirmon_entry.fail(worker_name, e)
|
|
90
92
|
dirmon_entry.save(validate: false)
|
|
91
93
|
end
|
|
@@ -105,7 +105,7 @@ module RocketJob
|
|
|
105
105
|
|
|
106
106
|
# Add a new output category and collect output for it.
|
|
107
107
|
def add_output_category(**args)
|
|
108
|
-
|
|
108
|
+
output_categories << RocketJob::Category::Output.new(**args)
|
|
109
109
|
end
|
|
110
110
|
|
|
111
111
|
private
|
|
@@ -94,11 +94,12 @@ module RocketJob
|
|
|
94
94
|
end
|
|
95
95
|
|
|
96
96
|
VALID_INSTANCE_METHODS = %i[upload upload_file_name= full_file_name=].freeze
|
|
97
|
+
private_constant :VALID_INSTANCE_METHODS
|
|
97
98
|
|
|
98
99
|
# Validates job_class is a Rocket Job
|
|
99
100
|
def job_implements_upload
|
|
100
101
|
klass = job_class
|
|
101
|
-
return if klass.nil? || klass.instance_methods.
|
|
102
|
+
return if klass.nil? || klass.instance_methods.intersect?(VALID_INSTANCE_METHODS)
|
|
102
103
|
|
|
103
104
|
errors.add(:job_class_name,
|
|
104
105
|
"#{job_class} must implement any one of: :#{VALID_INSTANCE_METHODS.join(' :')} instance methods")
|
|
@@ -120,14 +121,16 @@ module RocketJob
|
|
|
120
121
|
klass = job_class
|
|
121
122
|
return unless klass
|
|
122
123
|
|
|
123
|
-
|
|
124
|
-
|
|
124
|
+
# Mongoid 9 returns Hash field keys as Strings, earlier versions as Symbols.
|
|
125
|
+
properties.each_pair do |raw_key, value|
|
|
126
|
+
k = raw_key.to_sym
|
|
127
|
+
next if klass.public_method_defined?(:"#{k}=")
|
|
125
128
|
|
|
126
129
|
if %i[output_categories input_categories].include?(k)
|
|
127
130
|
category_class = k == :input_categories ? RocketJob::Category::Input : RocketJob::Category::Output
|
|
128
|
-
|
|
131
|
+
value.each do |category|
|
|
129
132
|
category.each_pair do |key, _value|
|
|
130
|
-
next if category_class.public_method_defined?("#{key}="
|
|
133
|
+
next if category_class.public_method_defined?(:"#{key}=")
|
|
131
134
|
|
|
132
135
|
errors.add(
|
|
133
136
|
:properties,
|
|
@@ -12,8 +12,8 @@ module RocketJob
|
|
|
12
12
|
# end
|
|
13
13
|
#
|
|
14
14
|
# input_category(:my_lookup).find(id: 123).first
|
|
15
|
-
def upload(batch_size: 10_000, &
|
|
16
|
-
BatchUploader.upload(batch_size: batch_size, &
|
|
15
|
+
def upload(batch_size: 10_000, &)
|
|
16
|
+
BatchUploader.upload(batch_size: batch_size, &)
|
|
17
17
|
end
|
|
18
18
|
|
|
19
19
|
# Looks up the value at the specified id.
|
|
@@ -18,7 +18,7 @@ module RocketJob
|
|
|
18
18
|
# Loads the queue with jobs to be processed once the queue is loaded.
|
|
19
19
|
# Retain the first and last job for timings, all others are destroyed on completion.
|
|
20
20
|
def run_test_case(count = self.count)
|
|
21
|
-
if RocketJob::Server.where(:state.in => %w[running paused]).
|
|
21
|
+
if RocketJob::Server.where(:state.in => %w[running paused]).none?
|
|
22
22
|
raise "Please start servers before starting the performance test"
|
|
23
23
|
end
|
|
24
24
|
|
|
@@ -18,7 +18,8 @@ module RocketJob
|
|
|
18
18
|
|
|
19
19
|
# Whether to prevent another instance of this job from running with the exact _same_ cron schedule.
|
|
20
20
|
# Another job instance with a different `cron_schedule` string is permitted.
|
|
21
|
-
field :cron_singleton, type: Mongoid::Boolean, default: true, class_attribute: true, user_editable: true,
|
|
21
|
+
field :cron_singleton, type: Mongoid::Boolean, default: true, class_attribute: true, user_editable: true,
|
|
22
|
+
copy_on_restart: true
|
|
22
23
|
|
|
23
24
|
# Whether to re-schedule the next job occurrence when this job starts, or when it is complete.
|
|
24
25
|
#
|
|
@@ -38,7 +39,8 @@ module RocketJob
|
|
|
38
39
|
# at any time until after the job has failed, or is aborted.
|
|
39
40
|
# - To prevent this job creating any new duplicate instances during subsequent processing,
|
|
40
41
|
# its `cron_schedule` is set to `nil` after it fails or is aborted.
|
|
41
|
-
field :cron_after_start, type: Mongoid::Boolean, default: true, class_attribute: true, user_editable: true,
|
|
42
|
+
field :cron_after_start, type: Mongoid::Boolean, default: true, class_attribute: true, user_editable: true,
|
|
43
|
+
copy_on_restart: true
|
|
42
44
|
|
|
43
45
|
validates_each :cron_schedule do |record, attr, value|
|
|
44
46
|
record.errors.add(attr, "Invalid cron_schedule: #{value.inspect}") if value && !Fugit::Cron.new(value)
|
|
@@ -88,7 +90,8 @@ module RocketJob
|
|
|
88
90
|
def rocket_job_cron_singleton_check
|
|
89
91
|
return if cron_schedule.nil? || completed? || aborted? || !rocket_job_cron_duplicate?
|
|
90
92
|
|
|
91
|
-
errors.add(:state,
|
|
93
|
+
errors.add(:state,
|
|
94
|
+
"Another instance of #{self.class.name} is already queued, running, failed, or paused with the same cron schedule: #{cron_schedule}")
|
|
92
95
|
end
|
|
93
96
|
end
|
|
94
97
|
end
|
|
@@ -11,7 +11,6 @@ module RocketJob
|
|
|
11
11
|
store_in client: "rocketjob"
|
|
12
12
|
end
|
|
13
13
|
|
|
14
|
-
# rubocop:disable Style/RedundantSort
|
|
15
14
|
module ClassMethods
|
|
16
15
|
# Mongoid does not apply ordering, add sort
|
|
17
16
|
def first
|
|
@@ -23,7 +22,6 @@ module RocketJob
|
|
|
23
22
|
all.sort("_id" => -1).first
|
|
24
23
|
end
|
|
25
24
|
end
|
|
26
|
-
# rubocop:enable Style/RedundantSort
|
|
27
25
|
|
|
28
26
|
private
|
|
29
27
|
|
|
@@ -17,7 +17,7 @@ module RocketJob
|
|
|
17
17
|
# - log_exception logs entire exception if raised
|
|
18
18
|
# - on_exception_level changes log level from info to error on exception
|
|
19
19
|
# - silence noisy jobs by raising log level
|
|
20
|
-
def rocket_job_around_logger(&
|
|
20
|
+
def rocket_job_around_logger(&)
|
|
21
21
|
logger.info("Start #perform")
|
|
22
22
|
logger.measure_info(
|
|
23
23
|
"Completed #perform",
|
|
@@ -25,7 +25,7 @@ module RocketJob
|
|
|
25
25
|
log_exception: :full,
|
|
26
26
|
on_exception_level: :error,
|
|
27
27
|
silence: log_level,
|
|
28
|
-
&
|
|
28
|
+
&
|
|
29
29
|
)
|
|
30
30
|
end
|
|
31
31
|
end
|
|
@@ -79,8 +79,8 @@ module RocketJob
|
|
|
79
79
|
return
|
|
80
80
|
end
|
|
81
81
|
|
|
82
|
-
job_attrs = self.class.rocket_job_restart_attributes.
|
|
83
|
-
|
|
82
|
+
job_attrs = self.class.rocket_job_restart_attributes.to_h do |attr|
|
|
83
|
+
[attr, send(attr)]
|
|
84
84
|
end
|
|
85
85
|
job_attrs.merge!(overrides)
|
|
86
86
|
|
|
@@ -30,6 +30,13 @@ module RocketJob
|
|
|
30
30
|
|
|
31
31
|
included do
|
|
32
32
|
class_attribute :rocket_job_throttles
|
|
33
|
+
|
|
34
|
+
# Human readable reason why this job is currently being throttled.
|
|
35
|
+
# Set when a throttle prevents the job from running, cleared when it starts.
|
|
36
|
+
# Surfaced in Rocket Job Mission Control.
|
|
37
|
+
field :throttled_by, type: String
|
|
38
|
+
# Time at which this job was last throttled.
|
|
39
|
+
field :throttled_at, type: Time
|
|
33
40
|
end
|
|
34
41
|
|
|
35
42
|
module ClassMethods
|
|
@@ -45,11 +52,16 @@ module RocketJob
|
|
|
45
52
|
# Or, a block that will return the filter.
|
|
46
53
|
# Default: :throttle_filter_class (Throttle all jobs of this class)
|
|
47
54
|
#
|
|
55
|
+
# description: [String|Proc]
|
|
56
|
+
# Human readable reason why the job is throttled, persisted to `throttled_by`
|
|
57
|
+
# and shown in Mission Control. A Proc is called with the job and must
|
|
58
|
+
# return a String. Default: a humanized version of the method name.
|
|
59
|
+
#
|
|
48
60
|
# Note: Throttles are executed in the order they are defined.
|
|
49
|
-
def define_throttle(method_name, filter: :throttle_filter_class)
|
|
61
|
+
def define_throttle(method_name, filter: :throttle_filter_class, description: nil)
|
|
50
62
|
# Duplicate to prevent modifying parent class throttles
|
|
51
63
|
definitions = rocket_job_throttles ? rocket_job_throttles.deep_dup : ThrottleDefinitions.new
|
|
52
|
-
definitions.add(method_name, filter)
|
|
64
|
+
definitions.add(method_name, filter, description)
|
|
53
65
|
self.rocket_job_throttles = definitions
|
|
54
66
|
end
|
|
55
67
|
|
|
@@ -32,7 +32,10 @@ module RocketJob
|
|
|
32
32
|
# Allow jobs to be throttled by group name instance of the job class name.
|
|
33
33
|
field :throttle_group, type: String, class_attribute: true, user_editable: true, copy_on_restart: true
|
|
34
34
|
|
|
35
|
-
define_throttle :throttle_running_jobs_exceeded
|
|
35
|
+
define_throttle :throttle_running_jobs_exceeded?,
|
|
36
|
+
description: lambda { |job, *|
|
|
37
|
+
"Throttled: maximum of #{job.throttle_running_jobs} running jobs reached"
|
|
38
|
+
}
|
|
36
39
|
end
|
|
37
40
|
|
|
38
41
|
private
|
|
@@ -66,10 +66,13 @@ module RocketJob
|
|
|
66
66
|
# re_raise_exceptions: [true|false]
|
|
67
67
|
# Re-raise the exception after updating the job
|
|
68
68
|
# Default: false
|
|
69
|
-
def fail_on_exception!(re_raise_exceptions = false, &
|
|
70
|
-
SemanticLogger.named_tagged(job: id.to_s, &
|
|
69
|
+
def fail_on_exception!(re_raise_exceptions = false, &)
|
|
70
|
+
SemanticLogger.named_tagged(job: id.to_s, &)
|
|
71
71
|
rescue Exception => e
|
|
72
72
|
SemanticLogger.named_tagged(job: id.to_s) do
|
|
73
|
+
# Not a guard clause: the `fail` event clears `worker_name`, so the
|
|
74
|
+
# branches are not equivalent and must not be collapsed.
|
|
75
|
+
# rubocop:disable-next Style/GuardClause
|
|
73
76
|
if failed? || !may_fail?
|
|
74
77
|
self.exception = JobException.from_exception(e)
|
|
75
78
|
exception.worker_name = worker_name
|
|
@@ -126,7 +126,7 @@ module RocketJob
|
|
|
126
126
|
# Or, to see the total durations based on the number of retries:
|
|
127
127
|
# (0..count).map{|i| "#{i+1} ==> #{RocketJob.seconds_as_duration(intervals[0..i].sum)}"}
|
|
128
128
|
def rocket_job_retry_seconds_to_delay
|
|
129
|
-
rocket_job_failure_count**4 + 5
|
|
129
|
+
(rocket_job_failure_count**4) + 5
|
|
130
130
|
end
|
|
131
131
|
end
|
|
132
132
|
end
|