rocketjob 6.0.0.rc1 → 6.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +164 -8
- data/lib/rocket_job/batch/categories.rb +25 -18
- data/lib/rocket_job/batch/io.rb +130 -130
- data/lib/rocket_job/batch/performance.rb +2 -2
- data/lib/rocket_job/batch/statistics.rb +2 -2
- data/lib/rocket_job/batch/throttle_running_workers.rb +1 -1
- data/lib/rocket_job/batch/worker.rb +14 -12
- data/lib/rocket_job/batch.rb +0 -1
- data/lib/rocket_job/category/base.rb +10 -7
- data/lib/rocket_job/category/input.rb +61 -1
- data/lib/rocket_job/category/output.rb +9 -0
- data/lib/rocket_job/cli.rb +1 -1
- data/lib/rocket_job/dirmon_entry.rb +1 -1
- data/lib/rocket_job/extensions/mongoid/contextual/mongo.rb +2 -2
- data/lib/rocket_job/extensions/rocket_job_adapter.rb +2 -2
- data/lib/rocket_job/job_exception.rb +1 -1
- data/lib/rocket_job/jobs/conversion_job.rb +43 -0
- data/lib/rocket_job/jobs/dirmon_job.rb +24 -35
- data/lib/rocket_job/jobs/housekeeping_job.rb +4 -5
- data/lib/rocket_job/jobs/on_demand_batch_job.rb +15 -11
- data/lib/rocket_job/jobs/on_demand_job.rb +2 -2
- data/lib/rocket_job/jobs/re_encrypt/relational_job.rb +103 -97
- data/lib/rocket_job/jobs/upload_file_job.rb +6 -3
- data/lib/rocket_job/lookup_collection.rb +4 -3
- data/lib/rocket_job/plugins/cron.rb +60 -20
- data/lib/rocket_job/plugins/job/persistence.rb +36 -0
- data/lib/rocket_job/plugins/job/throttle.rb +2 -2
- data/lib/rocket_job/plugins/restart.rb +3 -110
- data/lib/rocket_job/plugins/state_machine.rb +2 -2
- data/lib/rocket_job/plugins/throttle_dependent_jobs.rb +43 -0
- data/lib/rocket_job/sliced/bzip2_output_slice.rb +18 -19
- data/lib/rocket_job/sliced/compressed_slice.rb +3 -6
- data/lib/rocket_job/sliced/encrypted_bzip2_output_slice.rb +49 -0
- data/lib/rocket_job/sliced/encrypted_slice.rb +4 -6
- data/lib/rocket_job/sliced/input.rb +42 -54
- data/lib/rocket_job/sliced/slice.rb +7 -3
- data/lib/rocket_job/sliced/slices.rb +12 -9
- data/lib/rocket_job/sliced/writer/input.rb +46 -18
- data/lib/rocket_job/sliced/writer/output.rb +0 -1
- data/lib/rocket_job/sliced.rb +1 -19
- data/lib/rocket_job/throttle_definitions.rb +7 -1
- data/lib/rocket_job/version.rb +1 -1
- data/lib/rocketjob.rb +4 -5
- metadata +12 -12
- data/lib/rocket_job/batch/tabular/input.rb +0 -133
- data/lib/rocket_job/batch/tabular/output.rb +0 -67
- data/lib/rocket_job/batch/tabular.rb +0 -58
@@ -1,6 +1,3 @@
|
|
1
|
-
require "active_record"
|
2
|
-
require "sync_attr"
|
3
|
-
|
4
1
|
# Batch Worker to Re-encrypt all encrypted fields in MySQL that start with `encrytped_`.
|
5
2
|
#
|
6
3
|
# Run in Rails console:
|
@@ -11,117 +8,126 @@ require "sync_attr"
|
|
11
8
|
# * This job will find any column in the database that starts with`encrypted_`.
|
12
9
|
# * This means that temporary or other tables not part of the application tables will also be processed.
|
13
10
|
# * Since it automatically finds and re-encrypts any column, new columns are handled without any manual intervention.
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
11
|
+
if defined?(ActiveRecord) && defined?(SyncAttr)
|
12
|
+
require "active_record"
|
13
|
+
require "sync_attr"
|
14
|
+
|
15
|
+
module RocketJob
|
16
|
+
module Jobs
|
17
|
+
module ReEncrypt
|
18
|
+
class RelationalJob < RocketJob::Job
|
19
|
+
include RocketJob::Batch
|
20
|
+
|
21
|
+
self.priority = 30
|
22
|
+
self.destroy_on_complete = false
|
23
|
+
self.throttle_running_jobs = 1
|
24
|
+
self.throttle_running_workers = 10
|
25
|
+
|
26
|
+
input_category slice_size: 1_000
|
27
|
+
|
28
|
+
# Name of the table being re-encrypted
|
29
|
+
field :table_name, type: String
|
30
|
+
|
31
|
+
# Limit the number of records to re-encrypt in test environments
|
32
|
+
field :limit, type: Integer
|
33
|
+
|
34
|
+
validates_presence_of :table_name
|
35
|
+
before_batch :upload_records
|
36
|
+
|
37
|
+
# Returns [Hash] of table names with each entry being an array
|
38
|
+
# of columns that start with encrypted_
|
39
|
+
sync_cattr_reader :encrypted_columns do
|
40
|
+
h = {}
|
41
|
+
connection.tables.each do |table|
|
42
|
+
columns = connection.columns(table)
|
43
|
+
columns.each do |column|
|
44
|
+
if column.name.start_with?("encrypted_")
|
45
|
+
add_column = column.name
|
46
|
+
(h[table] ||= []) << add_column if add_column
|
47
|
+
end
|
46
48
|
end
|
47
49
|
end
|
50
|
+
h
|
48
51
|
end
|
49
|
-
h
|
50
|
-
end
|
51
52
|
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
53
|
+
# Re-encrypt all `encrypted_` columns in the relational database.
|
54
|
+
# Queues a Job for each table that needs re-encryption.
|
55
|
+
def self.start(**args)
|
56
|
+
encrypted_columns.keys.collect do |table|
|
57
|
+
create!(table_name: table, description: table, **args)
|
58
|
+
end
|
57
59
|
end
|
58
|
-
end
|
59
60
|
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
61
|
+
# Re-encrypt all encrypted columns for the named table.
|
62
|
+
# Does not use AR models since we do not have models for all tables.
|
63
|
+
def perform(range)
|
64
|
+
start_id, end_id = range
|
64
65
|
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
66
|
+
columns = self.class.encrypted_columns[table_name]
|
67
|
+
unless columns&.size&.positive?
|
68
|
+
logger.error "No columns for table: #{table_name} from #{start_id} to #{end_id}"
|
69
|
+
return
|
70
|
+
end
|
70
71
|
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
72
|
+
logger.info "Processing: #{table_name} from #{start_id} to #{end_id}"
|
73
|
+
sql = "select id, #{columns.join(',')} from #{quoted_table_name} where id >= #{start_id} and id <= #{end_id}"
|
74
|
+
|
75
|
+
# Use AR to fetch all the records
|
76
|
+
self.class.connection.select_rows(sql).each do |row|
|
77
|
+
row.unshift(nil)
|
78
|
+
index = 1
|
79
|
+
sql = "update #{quoted_table_name} set "
|
80
|
+
updates = []
|
81
|
+
columns.collect do |column|
|
82
|
+
index += 1
|
83
|
+
value = row[index]
|
84
|
+
# Prevent re-encryption
|
85
|
+
unless value.blank?
|
86
|
+
new_value = re_encrypt(value)
|
87
|
+
updates << "#{column} = \"#{new_value}\"" if new_value != value
|
88
|
+
end
|
89
|
+
end
|
90
|
+
if updates.size.positive?
|
91
|
+
sql << updates.join(", ")
|
92
|
+
sql << " where id=#{row[1]}"
|
93
|
+
logger.trace sql
|
94
|
+
self.class.connection.execute sql
|
95
|
+
else
|
96
|
+
logger.trace { "Skipping empty values #{table_name}:#{row[1]}" }
|
87
97
|
end
|
88
|
-
end
|
89
|
-
if updates.size.positive?
|
90
|
-
sql << updates.join(", ")
|
91
|
-
sql << " where id=#{row[1]}"
|
92
|
-
logger.trace sql
|
93
|
-
self.class.connection.execute sql
|
94
|
-
else
|
95
|
-
logger.trace { "Skipping empty values #{table_name}:#{row[1]}" }
|
96
98
|
end
|
97
99
|
end
|
98
|
-
end
|
99
100
|
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
101
|
+
# Returns a database connection.
|
102
|
+
#
|
103
|
+
# Override this method to support other ways of obtaining a thread specific database connection.
|
104
|
+
def self.connection
|
105
|
+
ActiveRecord::Base.connection
|
106
|
+
end
|
106
107
|
|
107
|
-
|
108
|
+
private
|
108
109
|
|
109
|
-
|
110
|
-
|
111
|
-
|
110
|
+
def quoted_table_name
|
111
|
+
@quoted_table_name ||= self.class.connection.quote_table_name(table_name)
|
112
|
+
end
|
112
113
|
|
113
|
-
|
114
|
-
|
114
|
+
def re_encrypt(encrypted_value)
|
115
|
+
return encrypted_value if (encrypted_value == "") || encrypted_value.nil?
|
115
116
|
|
116
|
-
|
117
|
-
|
117
|
+
SymmetricEncryption.encrypt(SymmetricEncryption.decrypt(encrypted_value))
|
118
|
+
end
|
118
119
|
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
120
|
+
# Upload range to re-encrypt all rows in the specified table.
|
121
|
+
def upload_records
|
122
|
+
start_id = self.class.connection.select_value("select min(id) from #{quoted_table_name}").to_i
|
123
|
+
last_id = self.class.connection.select_value("select max(id) from #{quoted_table_name}").to_i
|
124
|
+
self.record_count =
|
125
|
+
if last_id.positive?
|
126
|
+
input.upload_integer_range_in_reverse_order(start_id, last_id) * input_category.slice_size
|
127
|
+
else
|
128
|
+
0
|
129
|
+
end
|
130
|
+
end
|
125
131
|
end
|
126
132
|
end
|
127
133
|
end
|
@@ -57,6 +57,10 @@ module RocketJob
|
|
57
57
|
|
58
58
|
def upload_file(job)
|
59
59
|
if job.respond_to?(:upload)
|
60
|
+
# Return the database connection for this thread back to the connection pool
|
61
|
+
# in case the upload takes a long time and the database connection expires.
|
62
|
+
ActiveRecord::Base.clear_active_connections! if defined?(ActiveRecord::Base)
|
63
|
+
|
60
64
|
if original_file_name
|
61
65
|
job.upload(upload_file_name, file_name: original_file_name)
|
62
66
|
else
|
@@ -96,11 +100,10 @@ module RocketJob
|
|
96
100
|
def file_exists
|
97
101
|
# Only check for file existence when it is a local file
|
98
102
|
return unless upload_file_name.is_a?(IOStreams::Paths::File)
|
99
|
-
if upload_file_name.to_s == ""
|
100
|
-
return errors.add(:upload_file_name, "Upload file name can't be blank.")
|
101
|
-
end
|
103
|
+
return errors.add(:upload_file_name, "Upload file name can't be blank.") if upload_file_name.to_s == ""
|
102
104
|
|
103
105
|
return if upload_file_name.exist?
|
106
|
+
|
104
107
|
errors.add(:upload_file_name, "Upload file: #{upload_file_name} does not exist.")
|
105
108
|
rescue NotImplementedError
|
106
109
|
nil
|
@@ -22,8 +22,6 @@ module RocketJob
|
|
22
22
|
find(id: id).first
|
23
23
|
end
|
24
24
|
|
25
|
-
private
|
26
|
-
|
27
25
|
# Internal class for uploading records in batches
|
28
26
|
class BatchUploader
|
29
27
|
attr_reader :record_count
|
@@ -46,7 +44,10 @@ module RocketJob
|
|
46
44
|
|
47
45
|
def <<(record)
|
48
46
|
raise(ArgumentError, "Record must be a Hash") unless record.is_a?(Hash)
|
49
|
-
|
47
|
+
|
48
|
+
unless record.key?(:id) || record.key?("id") || record.key?("_id")
|
49
|
+
raise(ArgumentError, "Record must include an :id key")
|
50
|
+
end
|
50
51
|
|
51
52
|
@documents << record
|
52
53
|
@record_count += 1
|
@@ -14,41 +14,81 @@ module RocketJob
|
|
14
14
|
extend ActiveSupport::Concern
|
15
15
|
|
16
16
|
included do
|
17
|
-
include Restart
|
18
|
-
|
19
17
|
field :cron_schedule, type: String, class_attribute: true, user_editable: true, copy_on_restart: true
|
20
18
|
|
19
|
+
# Whether to prevent another instance of this job from running with the exact _same_ cron schedule.
|
20
|
+
# Another job instance with a different `cron_schedule` string is permitted.
|
21
|
+
field :cron_singleton, type: Mongoid::Boolean, default: true, class_attribute: true, user_editable: true, copy_on_restart: true
|
22
|
+
|
23
|
+
# Whether to re-schedule the next job occurrence when this job starts, or when it is complete.
|
24
|
+
#
|
25
|
+
# `true`: Create a new scheduled instance of this job after it has started. (Default)
|
26
|
+
# - Ensures that the next scheduled instance is not missed because the current instance is still running.
|
27
|
+
# - Any changes to fields marked with `copy_on_restart` of `true` will be saved to the new scheduled instance
|
28
|
+
# _only_ if they were changed during an `after_start` callback.
|
29
|
+
# Changes to these during other callbacks or during the `perform` will not be saved to the new scheduled
|
30
|
+
# instance.
|
31
|
+
# - To prevent this job creating any new duplicate instances during subsequent processing,
|
32
|
+
# its `cron_schedule` is set to `nil`.
|
33
|
+
#
|
34
|
+
# `false`: Create a new scheduled instance of this job on `fail`, or `abort`.
|
35
|
+
# - Prevents the next scheduled instance from running or being scheduled while the current instance is
|
36
|
+
# still running.
|
37
|
+
# - Any changes to fields marked with `copy_on_restart` of `true` will be saved to the new scheduled instance
|
38
|
+
# at any time until after the job has failed, or is aborted.
|
39
|
+
# - To prevent this job creating any new duplicate instances during subsequent processing,
|
40
|
+
# its `cron_schedule` is set to `nil` after it fails or is aborted.
|
41
|
+
field :cron_after_start, type: Mongoid::Boolean, default: true, class_attribute: true, user_editable: true, copy_on_restart: true
|
42
|
+
|
21
43
|
validates_each :cron_schedule do |record, attr, value|
|
22
44
|
record.errors.add(attr, "Invalid cron_schedule: #{value.inspect}") if value && !Fugit::Cron.new(value)
|
23
45
|
end
|
46
|
+
validate :rocket_job_cron_singleton_check
|
47
|
+
|
24
48
|
before_save :rocket_job_cron_set_run_at
|
25
49
|
|
26
|
-
|
50
|
+
after_start :rocket_job_cron_on_start
|
51
|
+
after_abort :rocket_job_cron_end_state
|
52
|
+
after_complete :rocket_job_cron_end_state
|
53
|
+
after_fail :rocket_job_cron_end_state
|
54
|
+
end
|
27
55
|
|
28
|
-
|
29
|
-
|
30
|
-
def rocket_job_restart_new_instance
|
31
|
-
return unless cron_schedule
|
56
|
+
def rocket_job_cron_set_run_at
|
57
|
+
return if cron_schedule.nil? || !(cron_schedule_changed? && !run_at_changed?)
|
32
58
|
|
33
|
-
|
34
|
-
|
59
|
+
self.run_at = Fugit::Cron.new(cron_schedule).next_time.to_utc_time
|
60
|
+
end
|
35
61
|
|
36
|
-
|
37
|
-
# - create a new instance scheduled to run in the future.
|
38
|
-
# - clear out the `cron_schedule` so this instance will not schedule another instance to run on completion.
|
39
|
-
# Overrides: RocketJob::Plugins::Restart#rocket_job_restart_abort
|
40
|
-
def rocket_job_restart_abort
|
41
|
-
return unless cron_schedule
|
62
|
+
private
|
42
63
|
|
43
|
-
|
44
|
-
|
64
|
+
def rocket_job_cron_on_start
|
65
|
+
return unless cron_schedule && cron_after_start
|
66
|
+
|
67
|
+
current_cron_schedule = cron_schedule
|
68
|
+
update_attribute(:cron_schedule, nil)
|
69
|
+
create_restart!(cron_schedule: current_cron_schedule)
|
70
|
+
end
|
71
|
+
|
72
|
+
def rocket_job_cron_end_state
|
73
|
+
return unless cron_schedule && !cron_after_start
|
74
|
+
|
75
|
+
current_cron_schedule = cron_schedule
|
76
|
+
update_attribute(:cron_schedule, nil)
|
77
|
+
create_restart!(cron_schedule: current_cron_schedule)
|
78
|
+
end
|
79
|
+
|
80
|
+
# Returns [true|false] whether another instance of this job with the same cron schedule is already active
|
81
|
+
def rocket_job_cron_duplicate?
|
82
|
+
self.class.with(read: {mode: :primary}) do |conn|
|
83
|
+
conn.where(:state.in => %i[queued running failed paused], :id.ne => id, cron_schedule: cron_schedule).exists?
|
45
84
|
end
|
46
85
|
end
|
47
86
|
|
48
|
-
|
49
|
-
|
87
|
+
# Prevent creation of a new job when another is running with the same cron schedule.
|
88
|
+
def rocket_job_cron_singleton_check
|
89
|
+
return if cron_schedule.nil? || completed? || aborted? || !rocket_job_cron_duplicate?
|
50
90
|
|
51
|
-
self.
|
91
|
+
errors.add(:state, "Another instance of #{self.class.name} is already queued, running, failed, or paused with the same cron schedule: #{cron_schedule}")
|
52
92
|
end
|
53
93
|
end
|
54
94
|
end
|
@@ -70,6 +70,29 @@ module RocketJob
|
|
70
70
|
end
|
71
71
|
end
|
72
72
|
|
73
|
+
# Create a new instance of this job, copying across only the `copy_on_restart` attributes.
|
74
|
+
# Copy across input and output categories to new scheduled job so that all of the
|
75
|
+
# settings are remembered between instance. Example: slice_size
|
76
|
+
def create_restart!(**overrides)
|
77
|
+
if expired?
|
78
|
+
logger.info("Job has expired. Not creating a new instance.")
|
79
|
+
return
|
80
|
+
end
|
81
|
+
|
82
|
+
job_attrs = self.class.rocket_job_restart_attributes.each_with_object({}) do |attr, attrs|
|
83
|
+
attrs[attr] = send(attr)
|
84
|
+
end
|
85
|
+
job_attrs.merge!(overrides)
|
86
|
+
|
87
|
+
job = self.class.new(job_attrs)
|
88
|
+
job.input_categories = input_categories if respond_to?(:input_categories)
|
89
|
+
job.output_categories = output_categories if respond_to?(:output_categories)
|
90
|
+
|
91
|
+
job.save_with_retry!
|
92
|
+
|
93
|
+
logger.info("Created a new job instance: #{job.id}")
|
94
|
+
end
|
95
|
+
|
73
96
|
# Set in-memory job to complete if `destroy_on_complete` and the job has been destroyed
|
74
97
|
def reload
|
75
98
|
return super unless destroy_on_complete
|
@@ -85,6 +108,19 @@ module RocketJob
|
|
85
108
|
self
|
86
109
|
end
|
87
110
|
end
|
111
|
+
|
112
|
+
# Save with retry in case persistence takes a moment.
|
113
|
+
def save_with_retry!(retry_limit = 10, sleep_interval = 0.5)
|
114
|
+
count = 0
|
115
|
+
while count < retry_limit
|
116
|
+
return true if save
|
117
|
+
|
118
|
+
logger.info("Retrying to persist new scheduled instance: #{errors.messages.inspect}")
|
119
|
+
sleep(sleep_interval)
|
120
|
+
count += 1
|
121
|
+
end
|
122
|
+
save!
|
123
|
+
end
|
88
124
|
end
|
89
125
|
end
|
90
126
|
end
|
@@ -48,7 +48,7 @@ module RocketJob
|
|
48
48
|
# Note: Throttles are executed in the order they are defined.
|
49
49
|
def define_throttle(method_name, filter: :throttle_filter_class)
|
50
50
|
# Duplicate to prevent modifying parent class throttles
|
51
|
-
definitions = rocket_job_throttles ? rocket_job_throttles.
|
51
|
+
definitions = rocket_job_throttles ? rocket_job_throttles.deep_dup : ThrottleDefinitions.new
|
52
52
|
definitions.add(method_name, filter)
|
53
53
|
self.rocket_job_throttles = definitions
|
54
54
|
end
|
@@ -57,7 +57,7 @@ module RocketJob
|
|
57
57
|
def undefine_throttle(method_name)
|
58
58
|
return unless rocket_job_throttles
|
59
59
|
|
60
|
-
definitions = rocket_job_throttles.
|
60
|
+
definitions = rocket_job_throttles.deep_dup
|
61
61
|
definitions.remove(method_name)
|
62
62
|
self.rocket_job_throttles = definitions
|
63
63
|
end
|
@@ -2,128 +2,21 @@ require "active_support/concern"
|
|
2
2
|
|
3
3
|
module RocketJob
|
4
4
|
module Plugins
|
5
|
-
#
|
6
|
-
#
|
7
|
-
# Notes:
|
8
|
-
# * Restartable jobs automatically abort if they fail. This prevents the failed job from being retried.
|
9
|
-
# - To disable this behavior, add the following empty method:
|
10
|
-
# def rocket_job_restart_abort
|
11
|
-
# end
|
12
|
-
# * On destroy this job is destroyed without starting a new instance.
|
13
|
-
# * On Abort a new instance is created.
|
14
|
-
# * Include `RocketJob::Plugins::Singleton` to prevent multiple copies of a job from running at
|
15
|
-
# the same time.
|
16
|
-
# * The job will not be restarted if:
|
17
|
-
# - A validation fails after creating the new instance of this job.
|
18
|
-
# - The job has expired.
|
19
|
-
# * Only the fields that have `copy_on_restart: true` will be passed onto the new instance of this job.
|
20
|
-
#
|
21
|
-
# Example:
|
22
|
-
#
|
23
|
-
# class RestartableJob < RocketJob::Job
|
24
|
-
# include RocketJob::Plugins::Restart
|
25
|
-
#
|
26
|
-
# # Retain the completed job under the completed tab in Rocket Job Web Interface.
|
27
|
-
# self.destroy_on_complete = false
|
28
|
-
#
|
29
|
-
# # Will be copied to the new job on restart.
|
30
|
-
# field :limit, type: Integer, copy_on_restart: true
|
31
|
-
#
|
32
|
-
# # Will _not_ be copied to the new job on restart.
|
33
|
-
# field :list, type: Array, default: [1,2,3]
|
34
|
-
#
|
35
|
-
# # Set run_at every time a new instance of the job is created.
|
36
|
-
# after_initialize set_run_at, if: :new_record?
|
37
|
-
#
|
38
|
-
# def perform
|
39
|
-
# puts "The limit is #{limit}"
|
40
|
-
# puts "The list is #{list}"
|
41
|
-
# 'DONE'
|
42
|
-
# end
|
43
|
-
#
|
44
|
-
# private
|
45
|
-
#
|
46
|
-
# # Run this job in 30 minutes.
|
47
|
-
# def set_run_at
|
48
|
-
# self.run_at = 30.minutes.from_now
|
49
|
-
# end
|
50
|
-
# end
|
51
|
-
#
|
52
|
-
# job = RestartableJob.create!(limit: 10, list: [4,5,6])
|
53
|
-
# job.reload.state
|
54
|
-
# # => :queued
|
55
|
-
#
|
56
|
-
# job.limit
|
57
|
-
# # => 10
|
58
|
-
#
|
59
|
-
# job.list
|
60
|
-
# # => [4,5,6]
|
61
|
-
#
|
62
|
-
# # Wait 30 minutes ...
|
63
|
-
#
|
64
|
-
# job.reload.state
|
65
|
-
# # => :completed
|
66
|
-
#
|
67
|
-
# # A new instance was automatically created.
|
68
|
-
# job2 = RestartableJob.last
|
69
|
-
# job2.state
|
70
|
-
# # => :queued
|
71
|
-
#
|
72
|
-
# job2.limit
|
73
|
-
# # => 10
|
74
|
-
#
|
75
|
-
# job2.list
|
76
|
-
# # => [1,2,3]
|
5
|
+
# @deprecated
|
77
6
|
module Restart
|
78
7
|
extend ActiveSupport::Concern
|
79
8
|
|
80
9
|
included do
|
81
|
-
after_abort :
|
82
|
-
after_complete :
|
10
|
+
after_abort :create_restart!
|
11
|
+
after_complete :create_restart!
|
83
12
|
after_fail :rocket_job_restart_abort
|
84
13
|
end
|
85
14
|
|
86
15
|
private
|
87
16
|
|
88
|
-
# Run again in the future, even if this run fails with an exception
|
89
|
-
def rocket_job_restart_new_instance
|
90
|
-
if expired?
|
91
|
-
logger.info("Job has expired. Not creating a new instance.")
|
92
|
-
return
|
93
|
-
end
|
94
|
-
job_attrs =
|
95
|
-
rocket_job_restart_attributes.each_with_object({}) { |attr, attrs| attrs[attr] = send(attr) }
|
96
|
-
job = self.class.new(job_attrs)
|
97
|
-
|
98
|
-
# Copy across input and output categories to new scheduled job so that all of the
|
99
|
-
# settings are remembered between instance. Example: slice_size
|
100
|
-
job.input_categories = input_categories if respond_to?(:input_categories)
|
101
|
-
job.output_categories = output_categories if respond_to?(:output_categories)
|
102
|
-
|
103
|
-
rocket_job_restart_save(job)
|
104
|
-
end
|
105
|
-
|
106
17
|
def rocket_job_restart_abort
|
107
18
|
new_record? ? abort : abort!
|
108
19
|
end
|
109
|
-
|
110
|
-
# Allow Singleton to prevent the creation of a new job if one is already running
|
111
|
-
# Retry since the delete may not have persisted to disk yet.
|
112
|
-
def rocket_job_restart_save(job, retry_limit = 10, sleep_interval = 0.5)
|
113
|
-
count = 0
|
114
|
-
while count < retry_limit
|
115
|
-
if job.save
|
116
|
-
logger.info("Created a new job instance: #{job.id}")
|
117
|
-
return true
|
118
|
-
else
|
119
|
-
logger.info("Job already active, retrying after a short sleep")
|
120
|
-
sleep(sleep_interval)
|
121
|
-
end
|
122
|
-
count += 1
|
123
|
-
end
|
124
|
-
logger.error("New job instance not started: #{job.errors.messages.inspect}")
|
125
|
-
false
|
126
|
-
end
|
127
20
|
end
|
128
21
|
end
|
129
22
|
end
|
@@ -36,8 +36,8 @@ module RocketJob
|
|
36
36
|
raise(ArgumentError, "Cannot supply both a method name and a block") if methods.size.positive? && block
|
37
37
|
raise(ArgumentError, "Must supply either a method name or a block") unless methods.size.positive? || block
|
38
38
|
|
39
|
-
#
|
40
|
-
# For example:
|
39
|
+
# Limitation with AASM. It only supports guards on event transitions, not for callbacks.
|
40
|
+
# For example, AASM does not support callback options such as :if and :unless, yet Rails callbacks do.
|
41
41
|
# before_start :my_callback, unless: :encrypted?
|
42
42
|
# before_start :my_callback, if: :encrypted?
|
43
43
|
event = aasm.state_machine.events[event_name]
|
@@ -0,0 +1,43 @@
|
|
1
|
+
require "active_support/concern"
|
2
|
+
module RocketJob
|
3
|
+
module Plugins
|
4
|
+
# Prevent this job from starting, or a batch slice from starting if the dependent jobs are running.
|
5
|
+
#
|
6
|
+
# Features:
|
7
|
+
# - Ensures dependent jobs won't run
|
8
|
+
# When the throttle has been exceeded all jobs of this class will be ignored until the
|
9
|
+
# next refresh. `RocketJob::Config::re_check_seconds` which by default is 60 seconds.
|
10
|
+
module ThrottleDependentJobs
|
11
|
+
extend ActiveSupport::Concern
|
12
|
+
|
13
|
+
included do
|
14
|
+
field :dependent_jobs, type: Array, class_attribute: true, user_editable: true, copy_on_restart: true
|
15
|
+
|
16
|
+
define_throttle :dependent_jobs_running?
|
17
|
+
define_batch_throttle :dependent_jobs_running? if respond_to?(:define_batch_throttle)
|
18
|
+
end
|
19
|
+
|
20
|
+
class_methods do
|
21
|
+
def depends_on_job(*jobs)
|
22
|
+
self.dependent_jobs = Array(jobs).collect(&:to_s)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
private
|
27
|
+
|
28
|
+
# Checks if there are any dependent jobs are running
|
29
|
+
def dependent_jobs_running?
|
30
|
+
return false if dependent_jobs.blank?
|
31
|
+
|
32
|
+
jobs_count = RocketJob::Job.running.where(:_type.in => dependent_jobs).count
|
33
|
+
return false if jobs_count.zero?
|
34
|
+
|
35
|
+
logger.info(
|
36
|
+
message: "#{jobs_count} Dependent Jobs are running from #{dependent_jobs.join(', ')}",
|
37
|
+
metric: "#{self.class.name}/dependent_jobs_throttle"
|
38
|
+
)
|
39
|
+
true
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|