rocketjob 6.3.2 → 7.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +38 -316
  3. data/bin/rocketjob_batch_perf +1 -1
  4. data/lib/rocket_job/batch/categories.rb +25 -25
  5. data/lib/rocket_job/batch/io.rb +21 -15
  6. data/lib/rocket_job/batch/logger.rb +2 -2
  7. data/lib/rocket_job/batch/model.rb +2 -2
  8. data/lib/rocket_job/batch/performance.rb +1 -1
  9. data/lib/rocket_job/batch/statistics.rb +9 -2
  10. data/lib/rocket_job/batch/throttle.rb +7 -2
  11. data/lib/rocket_job/batch/throttle_running_workers.rb +5 -1
  12. data/lib/rocket_job/batch/throttle_windows.rb +4 -2
  13. data/lib/rocket_job/batch/worker.rb +51 -11
  14. data/lib/rocket_job/config.rb +6 -1
  15. data/lib/rocket_job/dirmon_entry.rb +6 -4
  16. data/lib/rocket_job/event.rb +84 -4
  17. data/lib/rocket_job/extensions/mongo/logging.rb +1 -0
  18. data/lib/rocket_job/extensions/mongoid/clients/options.rb +3 -3
  19. data/lib/rocket_job/extensions/mongoid/contextual/mongo.rb +3 -4
  20. data/lib/rocket_job/extensions/mongoid/factory.rb +2 -2
  21. data/lib/rocket_job/extensions/psych/yaml_tree.rb +8 -4
  22. data/lib/rocket_job/extensions/rocket_job_adapter.rb +1 -1
  23. data/lib/rocket_job/jobs/active_job.rb +1 -1
  24. data/lib/rocket_job/jobs/dirmon_job.rb +3 -1
  25. data/lib/rocket_job/jobs/on_demand_batch_job.rb +1 -1
  26. data/lib/rocket_job/jobs/upload_file_job.rb +8 -5
  27. data/lib/rocket_job/lookup_collection.rb +2 -2
  28. data/lib/rocket_job/performance.rb +1 -1
  29. data/lib/rocket_job/plugins/cron.rb +6 -3
  30. data/lib/rocket_job/plugins/document.rb +0 -2
  31. data/lib/rocket_job/plugins/job/logger.rb +2 -2
  32. data/lib/rocket_job/plugins/job/model.rb +1 -1
  33. data/lib/rocket_job/plugins/job/persistence.rb +2 -2
  34. data/lib/rocket_job/plugins/job/throttle.rb +14 -2
  35. data/lib/rocket_job/plugins/job/throttle_running_jobs.rb +4 -1
  36. data/lib/rocket_job/plugins/job/worker.rb +5 -2
  37. data/lib/rocket_job/plugins/retry.rb +1 -1
  38. data/lib/rocket_job/plugins/throttle_dependent_jobs.rb +7 -2
  39. data/lib/rocket_job/plugins/transaction.rb +2 -2
  40. data/lib/rocket_job/sliced/input.rb +17 -12
  41. data/lib/rocket_job/sliced/slice.rb +3 -3
  42. data/lib/rocket_job/sliced/slices.rb +3 -6
  43. data/lib/rocket_job/supervisor.rb +2 -2
  44. data/lib/rocket_job/thread_worker.rb +3 -3
  45. data/lib/rocket_job/throttle_definition.rb +19 -2
  46. data/lib/rocket_job/throttle_definitions.rb +14 -9
  47. data/lib/rocket_job/version.rb +1 -1
  48. data/lib/rocket_job/worker.rb +77 -28
  49. data/lib/rocketjob.rb +1 -2
  50. metadata +33 -27
  51. data/lib/rocket_job/ractor_worker.rb +0 -42
@@ -36,7 +36,11 @@ module RocketJob
36
36
 
37
37
  validates :throttle_running_workers, numericality: {greater_than_or_equal_to: 0}, allow_nil: true
38
38
 
39
- define_batch_throttle :throttle_running_workers_exceeded?, filter: :throttle_filter_id
39
+ define_batch_throttle :throttle_running_workers_exceeded?,
40
+ filter: :throttle_filter_id,
41
+ description: lambda { |job, *|
42
+ "Throttled: maximum of #{job.throttle_running_workers} running workers reached"
43
+ }
40
44
  end
41
45
 
42
46
  private
@@ -43,7 +43,9 @@ module RocketJob
43
43
  # Duration in seconds of the secondary window.
44
44
  field :secondary_duration, type: Integer, class_attribute: true, user_editable: true, copy_on_restart: true
45
45
 
46
- define_batch_throttle :throttle_windows_exceeded?, filter: :throttle_filter_id
46
+ define_batch_throttle :throttle_windows_exceeded?,
47
+ filter: :throttle_filter_id,
48
+ description: "Throttled: outside of its processing window"
47
49
 
48
50
  validates_each :primary_schedule, :secondary_schedule do |record, attr, value|
49
51
  record.errors.add(attr, "Invalid #{attr}: #{value.inspect}") if value && !Fugit::Cron.new(value)
@@ -62,7 +64,7 @@ module RocketJob
62
64
 
63
65
  def throttle_outside_window?(schedule, duration)
64
66
  cron = Fugit::Cron.new(schedule)
65
- time = Time.now.utc + 1
67
+ time = Time.now.getutc + 1
66
68
  # Add 1 second since right now could be the very beginning of the processing window.
67
69
  previous_time = cron.previous_time(time).to_utc_time
68
70
  previous_time + duration < time
@@ -69,7 +69,7 @@ module RocketJob
69
69
  # Note: The slice will be removed from processing when this method completes
70
70
  #
71
71
  # @deprecated Please open a ticket if you need this behavior.
72
- def work_first_slice(&block)
72
+ def work_first_slice(&)
73
73
  raise "#work_first_slice can only be called from within before_batch callbacks" unless sub_state == :before
74
74
 
75
75
  # TODO: Make these settings configurable
@@ -89,7 +89,7 @@ module RocketJob
89
89
 
90
90
  # TODO: Persist that the first slice is being processed by this worker
91
91
  slice.start
92
- rocket_job_process_slice(slice, &block)
92
+ rocket_job_process_slice(slice, &)
93
93
  end
94
94
 
95
95
  # Returns [Array<ActiveWorker>] All workers actively working on this job
@@ -111,15 +111,52 @@ module RocketJob
111
111
  end
112
112
 
113
113
  def rocket_job_batch_throttled?(slice, worker)
114
- filter = self.class.rocket_job_batch_throttles.matching_filter(self, slice)
115
- return false unless filter
114
+ throttle = self.class.rocket_job_batch_throttles.matching_throttle(self, slice)
115
+ unless throttle
116
+ rocket_job_clear_throttled
117
+ return false
118
+ end
116
119
 
117
120
  # Restore retrieved slice so that other workers can process it later.
118
121
  slice.set(worker_name: nil, state: :queued, started_at: nil)
119
- worker.add_to_current_filter(filter)
122
+ worker.add_to_current_filter(throttle.extract_filter(self, slice))
123
+ rocket_job_set_throttled(throttle.extract_description(self, slice))
120
124
  true
121
125
  end
122
126
 
127
+ # Record why this batch job's slices are being throttled, for visibility in
128
+ # Mission Control.
129
+ #
130
+ # Unlike a simple job, a running batch job document is shared by every worker
131
+ # and is deliberately not written on each poll (see Worker#find_and_assign_job).
132
+ # The reason is therefore persisted with a guarded conditional update so the
133
+ # shared document is only written when the reason actually changes, not on
134
+ # every throttled poll.
135
+ def rocket_job_set_throttled(reason)
136
+ return if throttled_by == reason
137
+
138
+ self.class.with(write: {w: 1}) do |query|
139
+ query.where(id: id, state: :running, :throttled_by.ne => reason).
140
+ update_all("$set" => {throttled_by: reason, throttled_at: Time.now})
141
+ end
142
+ self.throttled_by = reason
143
+ end
144
+
145
+ # Clear the throttle reason once slices are flowing again.
146
+ #
147
+ # Gated on the in-memory value so a worker that never throttled this job
148
+ # issues no database round trip per slice; only a worker that previously set
149
+ # a reason clears it, with a guarded update to avoid a redundant write.
150
+ def rocket_job_clear_throttled
151
+ return if throttled_by.nil?
152
+
153
+ self.class.with(write: {w: 1}) do |query|
154
+ query.where(id: id, :throttled_by.ne => nil).
155
+ update_all("$unset" => {throttled_by: "", throttled_at: ""})
156
+ end
157
+ self.throttled_by = nil
158
+ end
159
+
123
160
  # Process a single slice from Mongo
124
161
  # Once the slice has been successfully processed it will be removed from the input collection
125
162
  # Returns [Integer] the number of records successfully processed
@@ -151,7 +188,7 @@ module RocketJob
151
188
  if slice.processing_record_number.to_i > 1
152
189
  append = true
153
190
  logger.info("Resuming previously incomplete slice from record number #{slice.processing_record_number}")
154
- slice.records[slice.processing_record_number - 1..-1]
191
+ slice.records[(slice.processing_record_number - 1)..]
155
192
  else
156
193
  # Reprocess all records in this slice.
157
194
  slice.processing_record_number = 0
@@ -162,8 +199,11 @@ module RocketJob
162
199
  RocketJob::Sliced::Writer::Output.collect(self, input_slice: slice, append: append) do |writer|
163
200
  records.each do |record|
164
201
  slice.processing_record_number += 1
165
- SemanticLogger.named_tagged(record: slice.current_record_number) do
166
- writer << rocket_job_batch_perform(slice, record, &block)
202
+ # `current_record_number` reads Mongoid fields, so compute it once per
203
+ # record and reuse it for both the log tag and `#perform`.
204
+ record_number = slice.current_record_number
205
+ SemanticLogger.named_tagged(record: record_number) do
206
+ writer << rocket_job_batch_perform(slice, record, record_number, &block)
167
207
  count += 1
168
208
  end
169
209
  end
@@ -172,8 +212,8 @@ module RocketJob
172
212
  end
173
213
 
174
214
  # Perform a single record within the current slice.
175
- def rocket_job_batch_perform(slice, record)
176
- @rocket_job_record_number = slice.current_record_number
215
+ def rocket_job_batch_perform(slice, record, record_number = slice.current_record_number)
216
+ @rocket_job_record_number = record_number
177
217
 
178
218
  return block_given? ? yield(record) : perform(record) if _perform_callbacks.empty?
179
219
 
@@ -248,7 +288,7 @@ module RocketJob
248
288
  unless new_record?
249
289
  # Fail job iff no other worker has already finished it
250
290
  # Must set write concern to at least 1 since we need the nModified back
251
- result = self.class.with(write: {w: 1}) do |query|
291
+ result = self.class.with(write: {w: 1}) do |query|
252
292
  query.
253
293
  where(id: id, state: :running, sub_state: :processing).
254
294
  update({"$set" => {state: "failed", worker_name: worker_name}})
@@ -81,6 +81,11 @@ module RocketJob
81
81
  logger.debug "Reading Mongo configuration from: #{config_file}"
82
82
  ::Mongoid.load!(config_file, environment)
83
83
 
84
+ # Mongoid 9 requires `Time.zone` to be set in order to mongoize Date and
85
+ # Time values. When running outside of Rails it may be unset, so default
86
+ # it here without overriding an explicit application setting.
87
+ ::Time.zone_default ||= ::ActiveSupport::TimeZone["UTC"]
88
+
84
89
  config_file =
85
90
  if encryption_file_name
86
91
  Pathname.new(encryption_file_name)
@@ -100,7 +105,7 @@ module RocketJob
100
105
  def self.filter
101
106
  raise(ArgumentError, "Cannot supply both an include_filter and an exclude_filter") if include_filter && exclude_filter
102
107
 
103
- filter = where_filter
108
+ filter = where_filter
104
109
  (filter ||= {})["_type"] = include_filter if include_filter
105
110
  (filter ||= {})["_type"] = {"$not" => exclude_filter} if exclude_filter
106
111
  filter&.dup
@@ -286,14 +286,16 @@ module RocketJob
286
286
  klass = job_class
287
287
  return unless klass
288
288
 
289
- properties.each_pair do |k, _v|
290
- next if klass.public_method_defined?("#{k}=".to_sym)
289
+ # Mongoid 9 returns Hash field keys as Strings, earlier versions as Symbols.
290
+ properties.each_pair do |raw_key, value|
291
+ k = raw_key.to_sym
292
+ next if klass.public_method_defined?(:"#{k}=")
291
293
 
292
294
  if %i[output_categories input_categories].include?(k)
293
295
  category_class = k == :input_categories ? RocketJob::Category::Input : RocketJob::Category::Output
294
- properties[k].each do |category|
296
+ value.each do |category|
295
297
  category.each_pair do |key, _value|
296
- next if category_class.public_method_defined?("#{key}=".to_sym)
298
+ next if category_class.public_method_defined?(:"#{key}=")
297
299
 
298
300
  errors.add(
299
301
  :properties,
@@ -12,6 +12,11 @@ module RocketJob
12
12
 
13
13
  ALL_EVENTS = "*".freeze
14
14
 
15
+ # MongoDB OperationFailure codes that are safe to ignore when concurrently
16
+ # creating the polling collection and its TTL index:
17
+ # 48 NamespaceExists, 85 IndexOptionsConflict, 86 IndexKeySpecsConflict.
18
+ IGNORABLE_CREATE_CODES = [48, 85, 86].freeze
19
+
15
20
  # Capped collection long polling interval.
16
21
  class_attribute :long_poll_seconds, instance_accessor: false
17
22
  self.long_poll_seconds = 300
@@ -23,6 +28,28 @@ module RocketJob
23
28
  class_attribute :capped_collection_size, instance_accessor: false
24
29
  self.capped_collection_size = 128 * 1024 * 1024
25
30
 
31
+ # Event listener strategy:
32
+ # :capped - Tail a capped collection with a tailable cursor (default).
33
+ # Lowest latency, but requires a data store that supports capped
34
+ # collections and tailable cursors (i.e. a real MongoDB server).
35
+ # :polling - Poll a regular collection for new events. Slightly higher latency
36
+ # (bounded by `poll_interval`), but works on any MongoDB-compatible
37
+ # store, including AWS DocumentDB which has no capped collections.
38
+ class_attribute :listener_strategy, instance_accessor: false
39
+ self.listener_strategy = :capped
40
+
41
+ # Polling strategy: seconds to wait between polls for new events.
42
+ class_attribute :poll_interval, instance_accessor: false
43
+ self.poll_interval = 1
44
+
45
+ # Polling strategy: seconds an event is retained before a TTL index removes it.
46
+ # Must comfortably exceed the longest expected listener downtime so that a
47
+ # restarting server can still recover events published while it was away.
48
+ #
49
+ # Default: 1 hour.
50
+ class_attribute :event_retention_seconds, instance_accessor: false
51
+ self.event_retention_seconds = 60 * 60
52
+
26
53
  # Mandatory Event Name
27
54
  # Examples:
28
55
  # '/rocket_job/config'
@@ -81,14 +108,23 @@ module RocketJob
81
108
  @subscribers.each_value { |v| v.delete_if { |i| i.object_id == handle } }
82
109
  end
83
110
 
84
- # Indefinitely tail the capped collection looking for new events.
111
+ # Indefinitely watch for new events, dispatching each to its subscribers.
85
112
  # time: the start time from which to start looking for new events.
86
113
  def self.listener(time: @load_time)
87
114
  Thread.current.name = "rocketjob event"
88
- create_capped_collection
89
115
 
90
- logger.info("Event listener started")
91
- tail_capped_collection(time) { |event| process_event(event) }
116
+ case listener_strategy
117
+ when :capped
118
+ create_capped_collection
119
+ logger.info("Event listener started (capped collection)")
120
+ tail_capped_collection(time) { |event| process_event(event) }
121
+ when :polling
122
+ create_polling_collection
123
+ logger.info("Event listener started (polling, interval: #{poll_interval}s)")
124
+ poll_collection(time) { |event| process_event(event) }
125
+ else
126
+ raise(ArgumentError, "Unknown RocketJob::Event.listener_strategy: #{listener_strategy.inspect}")
127
+ end
92
128
  rescue Exception => e
93
129
  logger.error("#listener Event listener is terminating due to unhandled exception", e)
94
130
  raise(e)
@@ -134,6 +170,50 @@ module RocketJob
134
170
  retry
135
171
  end
136
172
 
173
+ # Create the regular (non-capped) collection used by the polling strategy,
174
+ # along with a TTL index that expires old events.
175
+ #
176
+ # Safe to call repeatedly: it never drops data, and tolerates the collection
177
+ # and index already existing.
178
+ def self.create_polling_collection
179
+ collection.client[collection_name].create unless collection_exists?
180
+ collection.indexes.create_one({created_at: 1}, expire_after: event_retention_seconds)
181
+ rescue Mongo::Error::OperationFailure => e
182
+ # Ignore "collection already exists" and "index already exists" races between
183
+ # multiple servers starting at once. Anything else is a real error.
184
+ raise(e) unless IGNORABLE_CREATE_CODES.include?(e.code)
185
+ end
186
+
187
+ # Indefinitely poll a regular collection for new events.
188
+ # time: the start time from which to start looking for new events.
189
+ #
190
+ # After the first poll the `_id` of the last event seen is used as the
191
+ # watermark, which is monotonic and unique, so events sharing a `created_at`
192
+ # timestamp are never skipped.
193
+ def self.poll_collection(time, &block)
194
+ last_id = nil
195
+ loop do
196
+ last_id = poll_once(time, last_id, &block)
197
+ sleep(poll_interval)
198
+ end
199
+ rescue Mongo::Error::SocketError, Mongo::Error::SocketTimeoutError, Mongo::Error::OperationFailure, Timeout::Error => e
200
+ logger.info("Polling failed, retrying: #{e.class.name} #{e.message}")
201
+ sleep(poll_interval)
202
+ retry
203
+ end
204
+
205
+ # Perform a single polling pass, yielding every event newer than the watermark
206
+ # and returning the new watermark (the `_id` of the last event seen, or the
207
+ # supplied `last_id` when no new events were found).
208
+ def self.poll_once(time, last_id = nil)
209
+ filter = last_id ? {_id: {"$gt" => last_id}} : {created_at: {"$gt" => time}}
210
+ collection.find(filter).sort(_id: 1).each do |doc|
211
+ last_id = doc["_id"]
212
+ yield(Mongoid::Factory.from_db(Event, doc))
213
+ end
214
+ last_id
215
+ end
216
+
137
217
  # Process a new event, calling registered subscribers.
138
218
  def self.process_event(event)
139
219
  logger.info("Event Received", event.attributes)
@@ -4,6 +4,7 @@ module Mongo
4
4
  class Monitoring
5
5
  class CommandLogSubscriber
6
6
  include SemanticLogger::Loggable
7
+
7
8
  logger.name = "Mongo"
8
9
 
9
10
  undef :started
@@ -11,7 +11,7 @@ module RocketJob
11
11
  end
12
12
 
13
13
  def collection(parent = nil)
14
- @collection_name ? mongo_client[@collection_name] : super(parent)
14
+ @collection_name ? mongo_client[@collection_name] : super
15
15
  end
16
16
 
17
17
  def collection_name
@@ -31,5 +31,5 @@ module RocketJob
31
31
  end
32
32
  end
33
33
 
34
- ::Mongoid::Criteria.include(RocketJob::MongoidClients::Options)
35
- ::Mongoid::Document.include(RocketJob::MongoidClients::Options)
34
+ Mongoid::Criteria.include(RocketJob::MongoidClients::Options)
35
+ Mongoid::Document.include(RocketJob::MongoidClients::Options)
@@ -1,15 +1,14 @@
1
- ::Mongoid::Contextual::Mongo
2
1
  module Mongoid
3
2
  module Contextual
4
3
  class Mongo
5
4
  def initialize(criteria)
6
5
  @criteria = criteria
7
6
  @klass = criteria.klass
8
- @cache = criteria.options[:cache]
9
- # Only line changed is here, get collection name from criteria, not @klass
7
+ # Only line changed from the Mongoid implementation is here: fetch the
8
+ # collection from the criteria so a custom collection_name is honored
9
+ # rather than always using @klass.collection.
10
10
  # @collection = @klass.collection
11
11
  @collection = criteria.collection
12
-
13
12
  criteria.send(:merge_type_selection)
14
13
  @view = collection.find(criteria.selector, session: _session)
15
14
  apply_options
@@ -4,11 +4,11 @@ module RocketJob
4
4
  # Don't convert to Mongoid::Factory since it conflicts with Mongoid use.
5
5
  module MongoidFactory
6
6
  def from_db(klass, attributes = nil, criteria = nil, selected_fields = nil)
7
- obj = super(klass, attributes, criteria, selected_fields)
7
+ obj = super
8
8
  obj.collection_name = criteria.collection_name if criteria
9
9
  obj
10
10
  end
11
11
  end
12
12
  end
13
13
 
14
- ::Mongoid::Factory.extend(RocketJob::MongoidFactory)
14
+ Mongoid::Factory.extend(RocketJob::MongoidFactory)
@@ -1,8 +1,12 @@
1
1
  require "psych/visitors/yaml_tree"
2
2
 
3
- class Psych::Visitors::YAMLTree
4
- # Serialize IOStream path as a string
5
- def visit_IOStreams_Path(o)
6
- visit_String(o.to_s)
3
+ module Psych
4
+ module Visitors
5
+ class YAMLTree
6
+ # Serialize IOStream path as a string
7
+ def visit_IOStreams_Path(o)
8
+ visit_String(o.to_s)
9
+ end
10
+ end
7
11
  end
8
12
  end
@@ -4,7 +4,7 @@ module ActiveJob
4
4
  #
5
5
  # Ruby's missing batch system.
6
6
  #
7
- # Read more about Rocket Job {here}[http://rocketjob.io].
7
+ # Read more about Rocket Job {here}[https://rocketjob.reidmorrison.com].
8
8
  #
9
9
  # To use Rocket Job set the queue_adapter config to +:rocket_job+.
10
10
  #
@@ -1,7 +1,7 @@
1
1
  module RocketJob
2
2
  module Jobs
3
3
  # Used to wrap an Active Job
4
- class ActiveJob < RocketJob::Job #:nodoc:
4
+ class ActiveJob < RocketJob::Job # :nodoc:
5
5
  field :data, type: Hash
6
6
  field :active_job_id, type: String
7
7
  field :active_job_class, type: String
@@ -85,7 +85,9 @@ module RocketJob
85
85
  file_names[key] = size if size
86
86
  end
87
87
  rescue StandardError => e
88
- logger.error("Dirmon Entry: #{dirmon_entry.id} failed. Moved to `failed` state to prevent processing again without manual intervention.", e)
88
+ logger.error(
89
+ "Dirmon Entry: #{dirmon_entry.id} failed. Moved to `failed` state to prevent processing again without manual intervention.", e
90
+ )
89
91
  dirmon_entry.fail(worker_name, e)
90
92
  dirmon_entry.save(validate: false)
91
93
  end
@@ -105,7 +105,7 @@ module RocketJob
105
105
 
106
106
  # Add a new output category and collect output for it.
107
107
  def add_output_category(**args)
108
- self.output_categories << RocketJob::Category::Output.new(**args)
108
+ output_categories << RocketJob::Category::Output.new(**args)
109
109
  end
110
110
 
111
111
  private
@@ -94,11 +94,12 @@ module RocketJob
94
94
  end
95
95
 
96
96
  VALID_INSTANCE_METHODS = %i[upload upload_file_name= full_file_name=].freeze
97
+ private_constant :VALID_INSTANCE_METHODS
97
98
 
98
99
  # Validates job_class is a Rocket Job
99
100
  def job_implements_upload
100
101
  klass = job_class
101
- return if klass.nil? || klass.instance_methods.any? { |m| VALID_INSTANCE_METHODS.include?(m) }
102
+ return if klass.nil? || klass.instance_methods.intersect?(VALID_INSTANCE_METHODS)
102
103
 
103
104
  errors.add(:job_class_name,
104
105
  "#{job_class} must implement any one of: :#{VALID_INSTANCE_METHODS.join(' :')} instance methods")
@@ -120,14 +121,16 @@ module RocketJob
120
121
  klass = job_class
121
122
  return unless klass
122
123
 
123
- properties.each_pair do |k, _v|
124
- next if klass.public_method_defined?("#{k}=".to_sym)
124
+ # Mongoid 9 returns Hash field keys as Strings, earlier versions as Symbols.
125
+ properties.each_pair do |raw_key, value|
126
+ k = raw_key.to_sym
127
+ next if klass.public_method_defined?(:"#{k}=")
125
128
 
126
129
  if %i[output_categories input_categories].include?(k)
127
130
  category_class = k == :input_categories ? RocketJob::Category::Input : RocketJob::Category::Output
128
- properties[k].each do |category|
131
+ value.each do |category|
129
132
  category.each_pair do |key, _value|
130
- next if category_class.public_method_defined?("#{key}=".to_sym)
133
+ next if category_class.public_method_defined?(:"#{key}=")
131
134
 
132
135
  errors.add(
133
136
  :properties,
@@ -12,8 +12,8 @@ module RocketJob
12
12
  # end
13
13
  #
14
14
  # input_category(:my_lookup).find(id: 123).first
15
- def upload(batch_size: 10_000, &block)
16
- BatchUploader.upload(batch_size: batch_size, &block)
15
+ def upload(batch_size: 10_000, &)
16
+ BatchUploader.upload(batch_size: batch_size, &)
17
17
  end
18
18
 
19
19
  # Looks up the value at the specified id.
@@ -18,7 +18,7 @@ module RocketJob
18
18
  # Loads the queue with jobs to be processed once the queue is loaded.
19
19
  # Retain the first and last job for timings, all others are destroyed on completion.
20
20
  def run_test_case(count = self.count)
21
- if RocketJob::Server.where(:state.in => %w[running paused]).count.zero?
21
+ if RocketJob::Server.where(:state.in => %w[running paused]).none?
22
22
  raise "Please start servers before starting the performance test"
23
23
  end
24
24
 
@@ -18,7 +18,8 @@ module RocketJob
18
18
 
19
19
  # Whether to prevent another instance of this job from running with the exact _same_ cron schedule.
20
20
  # Another job instance with a different `cron_schedule` string is permitted.
21
- field :cron_singleton, type: Mongoid::Boolean, default: true, class_attribute: true, user_editable: true, copy_on_restart: true
21
+ field :cron_singleton, type: Mongoid::Boolean, default: true, class_attribute: true, user_editable: true,
22
+ copy_on_restart: true
22
23
 
23
24
  # Whether to re-schedule the next job occurrence when this job starts, or when it is complete.
24
25
  #
@@ -38,7 +39,8 @@ module RocketJob
38
39
  # at any time until after the job has failed, or is aborted.
39
40
  # - To prevent this job creating any new duplicate instances during subsequent processing,
40
41
  # its `cron_schedule` is set to `nil` after it fails or is aborted.
41
- field :cron_after_start, type: Mongoid::Boolean, default: true, class_attribute: true, user_editable: true, copy_on_restart: true
42
+ field :cron_after_start, type: Mongoid::Boolean, default: true, class_attribute: true, user_editable: true,
43
+ copy_on_restart: true
42
44
 
43
45
  validates_each :cron_schedule do |record, attr, value|
44
46
  record.errors.add(attr, "Invalid cron_schedule: #{value.inspect}") if value && !Fugit::Cron.new(value)
@@ -88,7 +90,8 @@ module RocketJob
88
90
  def rocket_job_cron_singleton_check
89
91
  return if cron_schedule.nil? || completed? || aborted? || !rocket_job_cron_duplicate?
90
92
 
91
- errors.add(:state, "Another instance of #{self.class.name} is already queued, running, failed, or paused with the same cron schedule: #{cron_schedule}")
93
+ errors.add(:state,
94
+ "Another instance of #{self.class.name} is already queued, running, failed, or paused with the same cron schedule: #{cron_schedule}")
92
95
  end
93
96
  end
94
97
  end
@@ -11,7 +11,6 @@ module RocketJob
11
11
  store_in client: "rocketjob"
12
12
  end
13
13
 
14
- # rubocop:disable Style/RedundantSort
15
14
  module ClassMethods
16
15
  # Mongoid does not apply ordering, add sort
17
16
  def first
@@ -23,7 +22,6 @@ module RocketJob
23
22
  all.sort("_id" => -1).first
24
23
  end
25
24
  end
26
- # rubocop:enable Style/RedundantSort
27
25
 
28
26
  private
29
27
 
@@ -17,7 +17,7 @@ module RocketJob
17
17
  # - log_exception logs entire exception if raised
18
18
  # - on_exception_level changes log level from info to error on exception
19
19
  # - silence noisy jobs by raising log level
20
- def rocket_job_around_logger(&block)
20
+ def rocket_job_around_logger(&)
21
21
  logger.info("Start #perform")
22
22
  logger.measure_info(
23
23
  "Completed #perform",
@@ -25,7 +25,7 @@ module RocketJob
25
25
  log_exception: :full,
26
26
  on_exception_level: :error,
27
27
  silence: log_level,
28
- &block
28
+ &
29
29
  )
30
30
  end
31
31
  end
@@ -185,7 +185,7 @@ module RocketJob
185
185
  self.rocket_job_restart_attributes += [name.to_sym]
186
186
  end
187
187
 
188
- super(name, options)
188
+ super
189
189
  end
190
190
 
191
191
  # Builds this job instance from the supplied properties hash.
@@ -79,8 +79,8 @@ module RocketJob
79
79
  return
80
80
  end
81
81
 
82
- job_attrs = self.class.rocket_job_restart_attributes.each_with_object({}) do |attr, attrs|
83
- attrs[attr] = send(attr)
82
+ job_attrs = self.class.rocket_job_restart_attributes.to_h do |attr|
83
+ [attr, send(attr)]
84
84
  end
85
85
  job_attrs.merge!(overrides)
86
86
 
@@ -30,6 +30,13 @@ module RocketJob
30
30
 
31
31
  included do
32
32
  class_attribute :rocket_job_throttles
33
+
34
+ # Human readable reason why this job is currently being throttled.
35
+ # Set when a throttle prevents the job from running, cleared when it starts.
36
+ # Surfaced in Rocket Job Mission Control.
37
+ field :throttled_by, type: String
38
+ # Time at which this job was last throttled.
39
+ field :throttled_at, type: Time
33
40
  end
34
41
 
35
42
  module ClassMethods
@@ -45,11 +52,16 @@ module RocketJob
45
52
  # Or, a block that will return the filter.
46
53
  # Default: :throttle_filter_class (Throttle all jobs of this class)
47
54
  #
55
+ # description: [String|Proc]
56
+ # Human readable reason why the job is throttled, persisted to `throttled_by`
57
+ # and shown in Mission Control. A Proc is called with the job and must
58
+ # return a String. Default: a humanized version of the method name.
59
+ #
48
60
  # Note: Throttles are executed in the order they are defined.
49
- def define_throttle(method_name, filter: :throttle_filter_class)
61
+ def define_throttle(method_name, filter: :throttle_filter_class, description: nil)
50
62
  # Duplicate to prevent modifying parent class throttles
51
63
  definitions = rocket_job_throttles ? rocket_job_throttles.deep_dup : ThrottleDefinitions.new
52
- definitions.add(method_name, filter)
64
+ definitions.add(method_name, filter, description)
53
65
  self.rocket_job_throttles = definitions
54
66
  end
55
67
 
@@ -32,7 +32,10 @@ module RocketJob
32
32
  # Allow jobs to be throttled by group name instance of the job class name.
33
33
  field :throttle_group, type: String, class_attribute: true, user_editable: true, copy_on_restart: true
34
34
 
35
- define_throttle :throttle_running_jobs_exceeded?
35
+ define_throttle :throttle_running_jobs_exceeded?,
36
+ description: lambda { |job, *|
37
+ "Throttled: maximum of #{job.throttle_running_jobs} running jobs reached"
38
+ }
36
39
  end
37
40
 
38
41
  private
@@ -66,10 +66,13 @@ module RocketJob
66
66
  # re_raise_exceptions: [true|false]
67
67
  # Re-raise the exception after updating the job
68
68
  # Default: false
69
- def fail_on_exception!(re_raise_exceptions = false, &block)
70
- SemanticLogger.named_tagged(job: id.to_s, &block)
69
+ def fail_on_exception!(re_raise_exceptions = false, &)
70
+ SemanticLogger.named_tagged(job: id.to_s, &)
71
71
  rescue Exception => e
72
72
  SemanticLogger.named_tagged(job: id.to_s) do
73
+ # Not a guard clause: the `fail` event clears `worker_name`, so the
74
+ # branches are not equivalent and must not be collapsed.
75
+ # rubocop:disable-next Style/GuardClause
73
76
  if failed? || !may_fail?
74
77
  self.exception = JobException.from_exception(e)
75
78
  exception.worker_name = worker_name
@@ -126,7 +126,7 @@ module RocketJob
126
126
  # Or, to see the total durations based on the number of retries:
127
127
  # (0..count).map{|i| "#{i+1} ==> #{RocketJob.seconds_as_duration(intervals[0..i].sum)}"}
128
128
  def rocket_job_retry_seconds_to_delay
129
- rocket_job_failure_count**4 + 5
129
+ (rocket_job_failure_count**4) + 5
130
130
  end
131
131
  end
132
132
  end