rocketjob 6.0.0.rc1 → 6.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +164 -8
  3. data/lib/rocket_job/batch/categories.rb +25 -18
  4. data/lib/rocket_job/batch/io.rb +130 -130
  5. data/lib/rocket_job/batch/performance.rb +2 -2
  6. data/lib/rocket_job/batch/statistics.rb +2 -2
  7. data/lib/rocket_job/batch/throttle_running_workers.rb +1 -1
  8. data/lib/rocket_job/batch/worker.rb +14 -12
  9. data/lib/rocket_job/batch.rb +0 -1
  10. data/lib/rocket_job/category/base.rb +10 -7
  11. data/lib/rocket_job/category/input.rb +61 -1
  12. data/lib/rocket_job/category/output.rb +9 -0
  13. data/lib/rocket_job/cli.rb +1 -1
  14. data/lib/rocket_job/dirmon_entry.rb +1 -1
  15. data/lib/rocket_job/extensions/mongoid/contextual/mongo.rb +2 -2
  16. data/lib/rocket_job/extensions/rocket_job_adapter.rb +2 -2
  17. data/lib/rocket_job/job_exception.rb +1 -1
  18. data/lib/rocket_job/jobs/conversion_job.rb +43 -0
  19. data/lib/rocket_job/jobs/dirmon_job.rb +24 -35
  20. data/lib/rocket_job/jobs/housekeeping_job.rb +4 -5
  21. data/lib/rocket_job/jobs/on_demand_batch_job.rb +15 -11
  22. data/lib/rocket_job/jobs/on_demand_job.rb +2 -2
  23. data/lib/rocket_job/jobs/re_encrypt/relational_job.rb +103 -97
  24. data/lib/rocket_job/jobs/upload_file_job.rb +6 -3
  25. data/lib/rocket_job/lookup_collection.rb +4 -3
  26. data/lib/rocket_job/plugins/cron.rb +60 -20
  27. data/lib/rocket_job/plugins/job/persistence.rb +36 -0
  28. data/lib/rocket_job/plugins/job/throttle.rb +2 -2
  29. data/lib/rocket_job/plugins/restart.rb +3 -110
  30. data/lib/rocket_job/plugins/state_machine.rb +2 -2
  31. data/lib/rocket_job/plugins/throttle_dependent_jobs.rb +43 -0
  32. data/lib/rocket_job/sliced/bzip2_output_slice.rb +18 -19
  33. data/lib/rocket_job/sliced/compressed_slice.rb +3 -6
  34. data/lib/rocket_job/sliced/encrypted_bzip2_output_slice.rb +49 -0
  35. data/lib/rocket_job/sliced/encrypted_slice.rb +4 -6
  36. data/lib/rocket_job/sliced/input.rb +42 -54
  37. data/lib/rocket_job/sliced/slice.rb +7 -3
  38. data/lib/rocket_job/sliced/slices.rb +12 -9
  39. data/lib/rocket_job/sliced/writer/input.rb +46 -18
  40. data/lib/rocket_job/sliced/writer/output.rb +0 -1
  41. data/lib/rocket_job/sliced.rb +1 -19
  42. data/lib/rocket_job/throttle_definitions.rb +7 -1
  43. data/lib/rocket_job/version.rb +1 -1
  44. data/lib/rocketjob.rb +4 -5
  45. metadata +12 -12
  46. data/lib/rocket_job/batch/tabular/input.rb +0 -133
  47. data/lib/rocket_job/batch/tabular/output.rb +0 -67
  48. data/lib/rocket_job/batch/tabular.rb +0 -58
@@ -1,6 +1,3 @@
1
- require "active_record"
2
- require "sync_attr"
3
-
4
1
  # Batch Worker to Re-encrypt all encrypted fields in MySQL that start with `encrytped_`.
5
2
  #
6
3
  # Run in Rails console:
@@ -11,117 +8,126 @@ require "sync_attr"
11
8
  # * This job will find any column in the database that starts with`encrypted_`.
12
9
  # * This means that temporary or other tables not part of the application tables will also be processed.
13
10
  # * Since it automatically finds and re-encrypts any column, new columns are handled without any manual intervention.
14
- module RocketJob
15
- module Jobs
16
- module ReEncrypt
17
- class RelationalJob < RocketJob::Job
18
- include RocketJob::Batch
19
-
20
- self.priority = 30
21
- self.destroy_on_complete = false
22
- self.throttle_running_jobs = 1
23
- self.throttle_running_workers = 10
24
-
25
- input_category slice_size: 1_000
26
-
27
- # Name of the table being re-encrypted
28
- field :table_name, type: String
29
-
30
- # Limit the number of records to re-encrypt in test environments
31
- field :limit, type: Integer
32
-
33
- validates_presence_of :table_name
34
- before_batch :upload_records
35
-
36
- # Returns [Hash] of table names with each entry being an array
37
- # of columns that start with encrypted_
38
- sync_cattr_reader :encrypted_columns do
39
- h = {}
40
- connection.tables.each do |table|
41
- columns = connection.columns(table)
42
- columns.each do |column|
43
- if column.name.start_with?("encrypted_")
44
- add_column = column.name
45
- (h[table] ||= []) << add_column if add_column
11
+ if defined?(ActiveRecord) && defined?(SyncAttr)
12
+ require "active_record"
13
+ require "sync_attr"
14
+
15
+ module RocketJob
16
+ module Jobs
17
+ module ReEncrypt
18
+ class RelationalJob < RocketJob::Job
19
+ include RocketJob::Batch
20
+
21
+ self.priority = 30
22
+ self.destroy_on_complete = false
23
+ self.throttle_running_jobs = 1
24
+ self.throttle_running_workers = 10
25
+
26
+ input_category slice_size: 1_000
27
+
28
+ # Name of the table being re-encrypted
29
+ field :table_name, type: String
30
+
31
+ # Limit the number of records to re-encrypt in test environments
32
+ field :limit, type: Integer
33
+
34
+ validates_presence_of :table_name
35
+ before_batch :upload_records
36
+
37
+ # Returns [Hash] of table names with each entry being an array
38
+ # of columns that start with encrypted_
39
+ sync_cattr_reader :encrypted_columns do
40
+ h = {}
41
+ connection.tables.each do |table|
42
+ columns = connection.columns(table)
43
+ columns.each do |column|
44
+ if column.name.start_with?("encrypted_")
45
+ add_column = column.name
46
+ (h[table] ||= []) << add_column if add_column
47
+ end
46
48
  end
47
49
  end
50
+ h
48
51
  end
49
- h
50
- end
51
52
 
52
- # Re-encrypt all `encrypted_` columns in the relational database.
53
- # Queues a Job for each table that needs re-encryption.
54
- def self.start(**args)
55
- encrypted_columns.keys.collect do |table|
56
- create!(table_name: table, description: table, **args)
53
+ # Re-encrypt all `encrypted_` columns in the relational database.
54
+ # Queues a Job for each table that needs re-encryption.
55
+ def self.start(**args)
56
+ encrypted_columns.keys.collect do |table|
57
+ create!(table_name: table, description: table, **args)
58
+ end
57
59
  end
58
- end
59
60
 
60
- # Re-encrypt all encrypted columns for the named table.
61
- # Does not use AR models since we do not have models for all tables.
62
- def perform(range)
63
- start_id, end_id = range
61
+ # Re-encrypt all encrypted columns for the named table.
62
+ # Does not use AR models since we do not have models for all tables.
63
+ def perform(range)
64
+ start_id, end_id = range
64
65
 
65
- columns = self.class.encrypted_columns[table_name]
66
- unless columns&.size&.positive?
67
- logger.error "No columns for table: #{table_name} from #{start_id} to #{end_id}"
68
- return
69
- end
66
+ columns = self.class.encrypted_columns[table_name]
67
+ unless columns&.size&.positive?
68
+ logger.error "No columns for table: #{table_name} from #{start_id} to #{end_id}"
69
+ return
70
+ end
70
71
 
71
- logger.info "Processing: #{table_name} from #{start_id} to #{end_id}"
72
- sql = "select id, #{columns.join(',')} from #{quoted_table_name} where id >= #{start_id} and id <= #{end_id}"
73
-
74
- # Use AR to fetch all the records
75
- self.class.connection.select_rows(sql).each do |row|
76
- row.unshift(nil)
77
- index = 1
78
- sql = "update #{quoted_table_name} set "
79
- updates = []
80
- columns.collect do |column|
81
- index += 1
82
- value = row[index]
83
- # Prevent re-encryption
84
- unless value.blank?
85
- new_value = re_encrypt(value)
86
- updates << "#{column} = \"#{new_value}\"" if new_value != value
72
+ logger.info "Processing: #{table_name} from #{start_id} to #{end_id}"
73
+ sql = "select id, #{columns.join(',')} from #{quoted_table_name} where id >= #{start_id} and id <= #{end_id}"
74
+
75
+ # Use AR to fetch all the records
76
+ self.class.connection.select_rows(sql).each do |row|
77
+ row.unshift(nil)
78
+ index = 1
79
+ sql = "update #{quoted_table_name} set "
80
+ updates = []
81
+ columns.collect do |column|
82
+ index += 1
83
+ value = row[index]
84
+ # Prevent re-encryption
85
+ unless value.blank?
86
+ new_value = re_encrypt(value)
87
+ updates << "#{column} = \"#{new_value}\"" if new_value != value
88
+ end
89
+ end
90
+ if updates.size.positive?
91
+ sql << updates.join(", ")
92
+ sql << " where id=#{row[1]}"
93
+ logger.trace sql
94
+ self.class.connection.execute sql
95
+ else
96
+ logger.trace { "Skipping empty values #{table_name}:#{row[1]}" }
87
97
  end
88
- end
89
- if updates.size.positive?
90
- sql << updates.join(", ")
91
- sql << " where id=#{row[1]}"
92
- logger.trace sql
93
- self.class.connection.execute sql
94
- else
95
- logger.trace { "Skipping empty values #{table_name}:#{row[1]}" }
96
98
  end
97
99
  end
98
- end
99
100
 
100
- # Returns a database connection.
101
- #
102
- # Override this method to support other ways of obtaining a thread specific database connection.
103
- def self.connection
104
- ActiveRecord::Base.connection
105
- end
101
+ # Returns a database connection.
102
+ #
103
+ # Override this method to support other ways of obtaining a thread specific database connection.
104
+ def self.connection
105
+ ActiveRecord::Base.connection
106
+ end
106
107
 
107
- private
108
+ private
108
109
 
109
- def quoted_table_name
110
- @quoted_table_name ||= self.class.connection.quote_table_name(table_name)
111
- end
110
+ def quoted_table_name
111
+ @quoted_table_name ||= self.class.connection.quote_table_name(table_name)
112
+ end
112
113
 
113
- def re_encrypt(encrypted_value)
114
- return encrypted_value if (encrypted_value == "") || encrypted_value.nil?
114
+ def re_encrypt(encrypted_value)
115
+ return encrypted_value if (encrypted_value == "") || encrypted_value.nil?
115
116
 
116
- SymmetricEncryption.encrypt(SymmetricEncryption.decrypt(encrypted_value))
117
- end
117
+ SymmetricEncryption.encrypt(SymmetricEncryption.decrypt(encrypted_value))
118
+ end
118
119
 
119
- # Upload range to re-encrypt all rows in the specified table.
120
- def upload_records
121
- start_id = self.class.connection.select_value("select min(id) from #{quoted_table_name}").to_i
122
- last_id = self.class.connection.select_value("select max(id) from #{quoted_table_name}").to_i
123
- self.record_count =
124
- last_id.positive? ? (input.upload_integer_range_in_reverse_order(start_id, last_id) * input_category.slice_size) : 0
120
+ # Upload range to re-encrypt all rows in the specified table.
121
+ def upload_records
122
+ start_id = self.class.connection.select_value("select min(id) from #{quoted_table_name}").to_i
123
+ last_id = self.class.connection.select_value("select max(id) from #{quoted_table_name}").to_i
124
+ self.record_count =
125
+ if last_id.positive?
126
+ input.upload_integer_range_in_reverse_order(start_id, last_id) * input_category.slice_size
127
+ else
128
+ 0
129
+ end
130
+ end
125
131
  end
126
132
  end
127
133
  end
@@ -57,6 +57,10 @@ module RocketJob
57
57
 
58
58
  def upload_file(job)
59
59
  if job.respond_to?(:upload)
60
+ # Return the database connection for this thread back to the connection pool
61
+ # in case the upload takes a long time and the database connection expires.
62
+ ActiveRecord::Base.clear_active_connections! if defined?(ActiveRecord::Base)
63
+
60
64
  if original_file_name
61
65
  job.upload(upload_file_name, file_name: original_file_name)
62
66
  else
@@ -96,11 +100,10 @@ module RocketJob
96
100
  def file_exists
97
101
  # Only check for file existence when it is a local file
98
102
  return unless upload_file_name.is_a?(IOStreams::Paths::File)
99
- if upload_file_name.to_s == ""
100
- return errors.add(:upload_file_name, "Upload file name can't be blank.")
101
- end
103
+ return errors.add(:upload_file_name, "Upload file name can't be blank.") if upload_file_name.to_s == ""
102
104
 
103
105
  return if upload_file_name.exist?
106
+
104
107
  errors.add(:upload_file_name, "Upload file: #{upload_file_name} does not exist.")
105
108
  rescue NotImplementedError
106
109
  nil
@@ -22,8 +22,6 @@ module RocketJob
22
22
  find(id: id).first
23
23
  end
24
24
 
25
- private
26
-
27
25
  # Internal class for uploading records in batches
28
26
  class BatchUploader
29
27
  attr_reader :record_count
@@ -46,7 +44,10 @@ module RocketJob
46
44
 
47
45
  def <<(record)
48
46
  raise(ArgumentError, "Record must be a Hash") unless record.is_a?(Hash)
49
- raise(ArgumentError, "Record must include an :id key") unless record.key?(:id) || record.key?("id") || record.key?("_id")
47
+
48
+ unless record.key?(:id) || record.key?("id") || record.key?("_id")
49
+ raise(ArgumentError, "Record must include an :id key")
50
+ end
50
51
 
51
52
  @documents << record
52
53
  @record_count += 1
@@ -14,41 +14,81 @@ module RocketJob
14
14
  extend ActiveSupport::Concern
15
15
 
16
16
  included do
17
- include Restart
18
-
19
17
  field :cron_schedule, type: String, class_attribute: true, user_editable: true, copy_on_restart: true
20
18
 
19
+ # Whether to prevent another instance of this job from running with the exact _same_ cron schedule.
20
+ # Another job instance with a different `cron_schedule` string is permitted.
21
+ field :cron_singleton, type: Mongoid::Boolean, default: true, class_attribute: true, user_editable: true, copy_on_restart: true
22
+
23
+ # Whether to re-schedule the next job occurrence when this job starts, or when it is complete.
24
+ #
25
+ # `true`: Create a new scheduled instance of this job after it has started. (Default)
26
+ # - Ensures that the next scheduled instance is not missed because the current instance is still running.
27
+ # - Any changes to fields marked with `copy_on_restart` of `true` will be saved to the new scheduled instance
28
+ # _only_ if they were changed during an `after_start` callback.
29
+ # Changes to these during other callbacks or during the `perform` will not be saved to the new scheduled
30
+ # instance.
31
+ # - To prevent this job creating any new duplicate instances during subsequent processing,
32
+ # its `cron_schedule` is set to `nil`.
33
+ #
34
+ # `false`: Create a new scheduled instance of this job on `fail`, or `abort`.
35
+ # - Prevents the next scheduled instance from running or being scheduled while the current instance is
36
+ # still running.
37
+ # - Any changes to fields marked with `copy_on_restart` of `true` will be saved to the new scheduled instance
38
+ # at any time until after the job has failed, or is aborted.
39
+ # - To prevent this job creating any new duplicate instances during subsequent processing,
40
+ # its `cron_schedule` is set to `nil` after it fails or is aborted.
41
+ field :cron_after_start, type: Mongoid::Boolean, default: true, class_attribute: true, user_editable: true, copy_on_restart: true
42
+
21
43
  validates_each :cron_schedule do |record, attr, value|
22
44
  record.errors.add(attr, "Invalid cron_schedule: #{value.inspect}") if value && !Fugit::Cron.new(value)
23
45
  end
46
+ validate :rocket_job_cron_singleton_check
47
+
24
48
  before_save :rocket_job_cron_set_run_at
25
49
 
26
- private
50
+ after_start :rocket_job_cron_on_start
51
+ after_abort :rocket_job_cron_end_state
52
+ after_complete :rocket_job_cron_end_state
53
+ after_fail :rocket_job_cron_end_state
54
+ end
27
55
 
28
- # Prevent auto restart if this job does not have a cron schedule.
29
- # Overrides: RocketJob::Plugins::Restart#rocket_job_restart_new_instance
30
- def rocket_job_restart_new_instance
31
- return unless cron_schedule
56
+ def rocket_job_cron_set_run_at
57
+ return if cron_schedule.nil? || !(cron_schedule_changed? && !run_at_changed?)
32
58
 
33
- super
34
- end
59
+ self.run_at = Fugit::Cron.new(cron_schedule).next_time.to_utc_time
60
+ end
35
61
 
36
- # On failure:
37
- # - create a new instance scheduled to run in the future.
38
- # - clear out the `cron_schedule` so this instance will not schedule another instance to run on completion.
39
- # Overrides: RocketJob::Plugins::Restart#rocket_job_restart_abort
40
- def rocket_job_restart_abort
41
- return unless cron_schedule
62
+ private
42
63
 
43
- rocket_job_restart_new_instance
44
- update_attribute(:cron_schedule, nil)
64
+ def rocket_job_cron_on_start
65
+ return unless cron_schedule && cron_after_start
66
+
67
+ current_cron_schedule = cron_schedule
68
+ update_attribute(:cron_schedule, nil)
69
+ create_restart!(cron_schedule: current_cron_schedule)
70
+ end
71
+
72
+ def rocket_job_cron_end_state
73
+ return unless cron_schedule && !cron_after_start
74
+
75
+ current_cron_schedule = cron_schedule
76
+ update_attribute(:cron_schedule, nil)
77
+ create_restart!(cron_schedule: current_cron_schedule)
78
+ end
79
+
80
+ # Returns [true|false] whether another instance of this job with the same cron schedule is already active
81
+ def rocket_job_cron_duplicate?
82
+ self.class.with(read: {mode: :primary}) do |conn|
83
+ conn.where(:state.in => %i[queued running failed paused], :id.ne => id, cron_schedule: cron_schedule).exists?
45
84
  end
46
85
  end
47
86
 
48
- def rocket_job_cron_set_run_at
49
- return if cron_schedule.nil? || !(cron_schedule_changed? && !run_at_changed?)
87
+ # Prevent creation of a new job when another is running with the same cron schedule.
88
+ def rocket_job_cron_singleton_check
89
+ return if cron_schedule.nil? || completed? || aborted? || !rocket_job_cron_duplicate?
50
90
 
51
- self.run_at = Fugit::Cron.new(cron_schedule).next_time.to_utc_time
91
+ errors.add(:state, "Another instance of #{self.class.name} is already queued, running, failed, or paused with the same cron schedule: #{cron_schedule}")
52
92
  end
53
93
  end
54
94
  end
@@ -70,6 +70,29 @@ module RocketJob
70
70
  end
71
71
  end
72
72
 
73
+ # Create a new instance of this job, copying across only the `copy_on_restart` attributes.
74
+ # Copy across input and output categories to new scheduled job so that all of the
75
+ # settings are remembered between instance. Example: slice_size
76
+ def create_restart!(**overrides)
77
+ if expired?
78
+ logger.info("Job has expired. Not creating a new instance.")
79
+ return
80
+ end
81
+
82
+ job_attrs = self.class.rocket_job_restart_attributes.each_with_object({}) do |attr, attrs|
83
+ attrs[attr] = send(attr)
84
+ end
85
+ job_attrs.merge!(overrides)
86
+
87
+ job = self.class.new(job_attrs)
88
+ job.input_categories = input_categories if respond_to?(:input_categories)
89
+ job.output_categories = output_categories if respond_to?(:output_categories)
90
+
91
+ job.save_with_retry!
92
+
93
+ logger.info("Created a new job instance: #{job.id}")
94
+ end
95
+
73
96
  # Set in-memory job to complete if `destroy_on_complete` and the job has been destroyed
74
97
  def reload
75
98
  return super unless destroy_on_complete
@@ -85,6 +108,19 @@ module RocketJob
85
108
  self
86
109
  end
87
110
  end
111
+
112
+ # Save with retry in case persistence takes a moment.
113
+ def save_with_retry!(retry_limit = 10, sleep_interval = 0.5)
114
+ count = 0
115
+ while count < retry_limit
116
+ return true if save
117
+
118
+ logger.info("Retrying to persist new scheduled instance: #{errors.messages.inspect}")
119
+ sleep(sleep_interval)
120
+ count += 1
121
+ end
122
+ save!
123
+ end
88
124
  end
89
125
  end
90
126
  end
@@ -48,7 +48,7 @@ module RocketJob
48
48
  # Note: Throttles are executed in the order they are defined.
49
49
  def define_throttle(method_name, filter: :throttle_filter_class)
50
50
  # Duplicate to prevent modifying parent class throttles
51
- definitions = rocket_job_throttles ? rocket_job_throttles.dup : ThrottleDefinitions.new
51
+ definitions = rocket_job_throttles ? rocket_job_throttles.deep_dup : ThrottleDefinitions.new
52
52
  definitions.add(method_name, filter)
53
53
  self.rocket_job_throttles = definitions
54
54
  end
@@ -57,7 +57,7 @@ module RocketJob
57
57
  def undefine_throttle(method_name)
58
58
  return unless rocket_job_throttles
59
59
 
60
- definitions = rocket_job_throttles.dup
60
+ definitions = rocket_job_throttles.deep_dup
61
61
  definitions.remove(method_name)
62
62
  self.rocket_job_throttles = definitions
63
63
  end
@@ -2,128 +2,21 @@ require "active_support/concern"
2
2
 
3
3
  module RocketJob
4
4
  module Plugins
5
- # Automatically starts a new instance of this job anytime it fails, aborts, or completes.
6
- #
7
- # Notes:
8
- # * Restartable jobs automatically abort if they fail. This prevents the failed job from being retried.
9
- # - To disable this behavior, add the following empty method:
10
- # def rocket_job_restart_abort
11
- # end
12
- # * On destroy this job is destroyed without starting a new instance.
13
- # * On Abort a new instance is created.
14
- # * Include `RocketJob::Plugins::Singleton` to prevent multiple copies of a job from running at
15
- # the same time.
16
- # * The job will not be restarted if:
17
- # - A validation fails after creating the new instance of this job.
18
- # - The job has expired.
19
- # * Only the fields that have `copy_on_restart: true` will be passed onto the new instance of this job.
20
- #
21
- # Example:
22
- #
23
- # class RestartableJob < RocketJob::Job
24
- # include RocketJob::Plugins::Restart
25
- #
26
- # # Retain the completed job under the completed tab in Rocket Job Web Interface.
27
- # self.destroy_on_complete = false
28
- #
29
- # # Will be copied to the new job on restart.
30
- # field :limit, type: Integer, copy_on_restart: true
31
- #
32
- # # Will _not_ be copied to the new job on restart.
33
- # field :list, type: Array, default: [1,2,3]
34
- #
35
- # # Set run_at every time a new instance of the job is created.
36
- # after_initialize set_run_at, if: :new_record?
37
- #
38
- # def perform
39
- # puts "The limit is #{limit}"
40
- # puts "The list is #{list}"
41
- # 'DONE'
42
- # end
43
- #
44
- # private
45
- #
46
- # # Run this job in 30 minutes.
47
- # def set_run_at
48
- # self.run_at = 30.minutes.from_now
49
- # end
50
- # end
51
- #
52
- # job = RestartableJob.create!(limit: 10, list: [4,5,6])
53
- # job.reload.state
54
- # # => :queued
55
- #
56
- # job.limit
57
- # # => 10
58
- #
59
- # job.list
60
- # # => [4,5,6]
61
- #
62
- # # Wait 30 minutes ...
63
- #
64
- # job.reload.state
65
- # # => :completed
66
- #
67
- # # A new instance was automatically created.
68
- # job2 = RestartableJob.last
69
- # job2.state
70
- # # => :queued
71
- #
72
- # job2.limit
73
- # # => 10
74
- #
75
- # job2.list
76
- # # => [1,2,3]
5
+ # @deprecated
77
6
  module Restart
78
7
  extend ActiveSupport::Concern
79
8
 
80
9
  included do
81
- after_abort :rocket_job_restart_new_instance
82
- after_complete :rocket_job_restart_new_instance
10
+ after_abort :create_restart!
11
+ after_complete :create_restart!
83
12
  after_fail :rocket_job_restart_abort
84
13
  end
85
14
 
86
15
  private
87
16
 
88
- # Run again in the future, even if this run fails with an exception
89
- def rocket_job_restart_new_instance
90
- if expired?
91
- logger.info("Job has expired. Not creating a new instance.")
92
- return
93
- end
94
- job_attrs =
95
- rocket_job_restart_attributes.each_with_object({}) { |attr, attrs| attrs[attr] = send(attr) }
96
- job = self.class.new(job_attrs)
97
-
98
- # Copy across input and output categories to new scheduled job so that all of the
99
- # settings are remembered between instance. Example: slice_size
100
- job.input_categories = input_categories if respond_to?(:input_categories)
101
- job.output_categories = output_categories if respond_to?(:output_categories)
102
-
103
- rocket_job_restart_save(job)
104
- end
105
-
106
17
  def rocket_job_restart_abort
107
18
  new_record? ? abort : abort!
108
19
  end
109
-
110
- # Allow Singleton to prevent the creation of a new job if one is already running
111
- # Retry since the delete may not have persisted to disk yet.
112
- def rocket_job_restart_save(job, retry_limit = 10, sleep_interval = 0.5)
113
- count = 0
114
- while count < retry_limit
115
- if job.save
116
- logger.info("Created a new job instance: #{job.id}")
117
- return true
118
- else
119
- logger.info("Job already active, retrying after a short sleep")
120
- sleep(sleep_interval)
121
- end
122
- count += 1
123
- end
124
- logger.error("New job instance not started: #{job.errors.messages.inspect}")
125
- false
126
- end
127
20
  end
128
21
  end
129
22
  end
@@ -36,8 +36,8 @@ module RocketJob
36
36
  raise(ArgumentError, "Cannot supply both a method name and a block") if methods.size.positive? && block
37
37
  raise(ArgumentError, "Must supply either a method name or a block") unless methods.size.positive? || block
38
38
 
39
- # TODO: Somehow get AASM to support options such as :if and :unless to be consistent with other callbacks
40
- # For example:
39
+ # Limitation with AASM. It only supports guards on event transitions, not for callbacks.
40
+ # For example, AASM does not support callback options such as :if and :unless, yet Rails callbacks do.
41
41
  # before_start :my_callback, unless: :encrypted?
42
42
  # before_start :my_callback, if: :encrypted?
43
43
  event = aasm.state_machine.events[event_name]
@@ -0,0 +1,43 @@
1
+ require "active_support/concern"
2
+ module RocketJob
3
+ module Plugins
4
+ # Prevent this job from starting, or a batch slice from starting if the dependent jobs are running.
5
+ #
6
+ # Features:
7
+ # - Ensures dependent jobs won't run
8
+ # When the throttle has been exceeded all jobs of this class will be ignored until the
9
+ # next refresh. `RocketJob::Config::re_check_seconds` which by default is 60 seconds.
10
+ module ThrottleDependentJobs
11
+ extend ActiveSupport::Concern
12
+
13
+ included do
14
+ field :dependent_jobs, type: Array, class_attribute: true, user_editable: true, copy_on_restart: true
15
+
16
+ define_throttle :dependent_jobs_running?
17
+ define_batch_throttle :dependent_jobs_running? if respond_to?(:define_batch_throttle)
18
+ end
19
+
20
+ class_methods do
21
+ def depends_on_job(*jobs)
22
+ self.dependent_jobs = Array(jobs).collect(&:to_s)
23
+ end
24
+ end
25
+
26
+ private
27
+
28
+ # Checks if there are any dependent jobs are running
29
+ def dependent_jobs_running?
30
+ return false if dependent_jobs.blank?
31
+
32
+ jobs_count = RocketJob::Job.running.where(:_type.in => dependent_jobs).count
33
+ return false if jobs_count.zero?
34
+
35
+ logger.info(
36
+ message: "#{jobs_count} Dependent Jobs are running from #{dependent_jobs.join(', ')}",
37
+ metric: "#{self.class.name}/dependent_jobs_throttle"
38
+ )
39
+ true
40
+ end
41
+ end
42
+ end
43
+ end