rocketjob 5.2.0.beta3 → 5.3.3

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 0cece68774d1d549fb219daa5f78f3def0e5b3c51d0659fea8812da200b6b268
4
- data.tar.gz: 69ca23691dcbe2c8ba1b7b5a60a4d18cb8b033cf37510c2a0f1f0a02075357a7
3
+ metadata.gz: d487b34621338794b4e3da43753d858bf174b45f4f0dff6c9f478e212a71db8e
4
+ data.tar.gz: b6c3ddf2688a54dee0ad6ae6d1c4e4fc38448674dcbb7e0052d0295e57258f2d
5
5
  SHA512:
6
- metadata.gz: ff6d1c42eca584b91dbf27312b962ed4615168616a77531d70ca9a252145aac47f1c0a08655433aa956b143bab6a517e0b973b63474ba25cb718eb018df3950d
7
- data.tar.gz: 9add07cf69e3bf6cfc0d1aa62902ce00deb07ba41f922491afbe531e00b364fddee70f99f86497b7ec45e97637e715d355e55a6c30dbebcde2a8811b907e4d03
6
+ metadata.gz: 83e61b6520e71e370b2519ee8f937bdf69060050e67ca4ec3146e6a9306a6c26fd0307cb54c94189c2149eb3f8e14fbe7aa8f3fc6f9c17d6b06804745ecaf570
7
+ data.tar.gz: 20d83702151f6a1333152aa7ee903e7f7c4a79593291496f80670b7952c170734edec16280f0f449db01483e77e7e3af48c8100905688a5bd8998da573f74160
data/README.md CHANGED
@@ -3,9 +3,9 @@
3
3
 
4
4
  Ruby's missing batch system
5
5
 
6
- Checkout http://rocketjob.io/
6
+ Checkout https://rocketjob.io/
7
7
 
8
- ![Rocket Job](http://rocketjob.io/images/rocket/rocket-icon-512x512.png)
8
+ ![Rocket Job](https://rocketjob.io/images/rocket/rocket-icon-512x512.png)
9
9
 
10
10
  ## Documentation
11
11
 
@@ -24,6 +24,7 @@ module RocketJob
24
24
  autoload :LowerPriority, "rocket_job/batch/lower_priority"
25
25
  autoload :Performance, "rocket_job/batch/performance"
26
26
  autoload :Statistics, "rocket_job/batch/statistics"
27
+ autoload :ThrottleWindows, "rocket_job/batch/throttle_windows"
27
28
  autoload :Result, "rocket_job/batch/result"
28
29
  autoload :Results, "rocket_job/batch/results"
29
30
  autoload :Tabular, "rocket_job/batch/tabular"
@@ -18,7 +18,7 @@ module RocketJob
18
18
  raise "Category #{category.inspect}, must be registered in input_categories: #{input_categories.inspect}"
19
19
  end
20
20
 
21
- (@inputs ||= {})[category] ||= RocketJob::Sliced::Input.new(rocket_job_io_slice_arguments("inputs", category))
21
+ (@inputs ||= {})[category] ||= RocketJob::Sliced::Input.new(**rocket_job_io_slice_arguments("inputs", category))
22
22
  end
23
23
 
24
24
  # Returns [RocketJob::Sliced::Output] output collection for holding output slices
@@ -34,7 +34,7 @@ module RocketJob
34
34
  raise "Category #{category.inspect}, must be registered in output_categories: #{output_categories.inspect}"
35
35
  end
36
36
 
37
- (@outputs ||= {})[category] ||= RocketJob::Sliced::Output.new(rocket_job_io_slice_arguments("outputs", category))
37
+ (@outputs ||= {})[category] ||= RocketJob::Sliced::Output.new(**rocket_job_io_slice_arguments("outputs", category))
38
38
  end
39
39
 
40
40
  # Upload the supplied file, io, IOStreams::Path, or IOStreams::Stream.
@@ -12,6 +12,7 @@ module RocketJob
12
12
  included do
13
13
  field :tabular_input_header, type: Array, class_attribute: true, user_editable: true
14
14
  field :tabular_input_format, type: Symbol, default: :csv, class_attribute: true, user_editable: true
15
+ field :tabular_input_options, type: Hash, class_attribute: true
15
16
 
16
17
  # tabular_input_mode: [:line | :array | :hash]
17
18
  # :line
@@ -53,7 +54,9 @@ module RocketJob
53
54
  input_stream = stream.nil? ? nil : IOStreams.new(stream)
54
55
 
55
56
  if stream && (tabular_input_type == :text)
56
- input_stream.option_or_stream(:encode, encoding: "UTF-8", cleaner: :printable, replace: "")
57
+ # Cannot change the length of fixed width lines
58
+ replace = tabular_input_format == :fixed ? " " : ""
59
+ input_stream.option_or_stream(:encode, encoding: "UTF-8", cleaner: :printable, replace: replace)
57
60
  end
58
61
 
59
62
  # If an input header is not required, then we don't extract it'
@@ -96,14 +99,15 @@ module RocketJob
96
99
  allowed_columns: tabular_input_white_list,
97
100
  required_columns: tabular_input_required,
98
101
  skip_unknown: tabular_input_skip_unknown,
99
- format: tabular_input_format
102
+ format: tabular_input_format,
103
+ format_options: tabular_input_options&.deep_symbolize_keys
100
104
  )
101
105
  end
102
106
 
103
107
  def tabular_input_render
104
- unless tabular_input_header.blank? && tabular_input.header?
105
- @rocket_job_input = tabular_input.record_parse(@rocket_job_input)
106
- end
108
+ return if tabular_input_header.blank? && tabular_input.header?
109
+
110
+ @rocket_job_input = tabular_input.record_parse(@rocket_job_input)
107
111
  end
108
112
 
109
113
  # Cleanse custom input header if supplied.
@@ -12,6 +12,7 @@ module RocketJob
12
12
  included do
13
13
  field :tabular_output_header, type: Array, class_attribute: true, user_editable: true, copy_on_restart: true
14
14
  field :tabular_output_format, type: Symbol, default: :csv, class_attribute: true, user_editable: true, copy_on_restart: true
15
+ field :tabular_output_options, type: Hash, class_attribute: true
15
16
 
16
17
  validates_inclusion_of :tabular_output_format, in: IOStreams::Tabular.registered_formats
17
18
 
@@ -31,8 +32,9 @@ module RocketJob
31
32
 
32
33
  # Overrides: `RocketJob::Batch::IO#download` to add the `tabular_output_header`.
33
34
  def download(file_name_or_io = nil, category: :main, **args, &block)
34
- # No header required
35
- return super(file_name_or_io, category: category, **args, &block) unless tabular_output.requires_header?(category)
35
+ unless tabular_output.requires_header?(category)
36
+ return super(file_name_or_io, category: category, **args, &block)
37
+ end
36
38
 
37
39
  header = tabular_output.render_header(category)
38
40
  super(file_name_or_io, header_line: header, category: category, **args, &block)
@@ -43,7 +45,11 @@ module RocketJob
43
45
  # Delimited instance used for this slice, by a single worker (thread)
44
46
  def tabular_output
45
47
  @tabular_output ||= Tabular.new(
46
- main: IOStreams::Tabular.new(columns: tabular_output_header, format: tabular_output_format)
48
+ main: IOStreams::Tabular.new(
49
+ columns: tabular_output_header,
50
+ format: tabular_output_format,
51
+ format_options: tabular_output_options&.deep_symbolize_keys
52
+ )
47
53
  )
48
54
  end
49
55
 
@@ -6,7 +6,7 @@ module RocketJob
6
6
  #
7
7
  # Example:
8
8
  # # Do not run any slices for this job when the MySQL slave delay exceeds 5 minutes.
9
- # class MyJob < RocketJob
9
+ # class MyJob < RocketJob::Job
10
10
  # include RocketJob::Batch
11
11
  #
12
12
  # # Define a custom mysql throttle
@@ -5,7 +5,7 @@ module RocketJob
5
5
  # Throttle the number of slices of a specific batch job that are processed at the same time.
6
6
  #
7
7
  # Example:
8
- # class MyJob < RocketJob
8
+ # class MyJob < RocketJob::Job
9
9
  # include RocketJob::Batch
10
10
  #
11
11
  # # Maximum number of slices to process at the same time for each running instance.
@@ -0,0 +1,72 @@
1
+ require "active_support/concern"
2
+ require "fugit"
3
+
4
+ module RocketJob
5
+ module Batch
6
+ # For a batch job that can run over a long period of time it can be useful
7
+ # to prevent its slices from being processed outside a predefined processing window.
8
+ #
9
+ # This plugin supports up to 2 different processing windows.
10
+ #
11
+ # For example, do not run this job during business hours.
12
+ # Allow it to run from 5pm until 8am the following day Mon through Fri.
13
+ #
14
+ # class AfterHoursJob < RocketJob::Job
15
+ # include RocketJob::Batch
16
+ # include RocketJob::Batch::ThrottleWindows
17
+ #
18
+ # # Monday through Thursday the job can start processing at 5pm Eastern.
19
+ # self.primary_schedule = "0 17 * * 1-4 America/New_York"
20
+ # # Slices are allowed to run until 8am the following day, which is 15 hours long:
21
+ # self.primary_duration = 15.hours
22
+ #
23
+ # # The slices for this job can run all weekend long, starting Friday at 5pm Eastern.
24
+ # self.secondary_schedule = "0 17 * * 5 America/New_York"
25
+ # # Slices are allowed to run until 8am on Monday morning, which is 63 hours long:
26
+ # self.secondary_duration = 63.hours
27
+ # end
28
+ #
29
+ # Notes:
30
+ # * These schedules do not affect when the job is started, completed, or when `before_batch` or
31
+ # `after_batch` processing is performed. It only limits when individual slices are processed.
32
+ module ThrottleWindows
33
+ extend ActiveSupport::Concern
34
+
35
+ included do
36
+ # Beginning of the primary schedule. In cron format, see Scheduled Jobs `cron_schedule` for examples.
37
+ field :primary_schedule, type: String, class_attribute: true, user_editable: true, copy_on_restart: true
38
+ # Duration in seconds of the primary window.
39
+ field :primary_duration, type: Integer, class_attribute: true, user_editable: true, copy_on_restart: true
40
+
41
+ # Beginning of the secondary schedule. In cron format, see Scheduled Jobs `cron_schedule` for examples.
42
+ field :secondary_schedule, type: String, class_attribute: true, user_editable: true, copy_on_restart: true
43
+ # Duration in seconds of the secondary window.
44
+ field :secondary_duration, type: Integer, class_attribute: true, user_editable: true, copy_on_restart: true
45
+
46
+ define_batch_throttle :throttle_windows_exceeded?, filter: :throttle_filter_id
47
+
48
+ validates_each :primary_schedule, :secondary_schedule do |record, attr, value|
49
+ record.errors.add(attr, "Invalid #{attr}: #{value.inspect}") if value && !Fugit::Cron.new(value)
50
+ end
51
+ end
52
+
53
+ private
54
+
55
+ def throttle_windows_exceeded?
56
+ exceeded = primary_schedule && primary_duration && throttle_outside_window?(primary_schedule, primary_duration)
57
+ if exceeded && secondary_schedule && secondary_duration
58
+ exceeded = throttle_outside_window?(secondary_schedule, secondary_duration)
59
+ end
60
+ exceeded
61
+ end
62
+
63
+ def throttle_outside_window?(schedule, duration)
64
+ cron = Fugit::Cron.new(schedule)
65
+ time = Time.now.utc + 1
66
+ # Add 1 second since right now could be the very beginning of the processing window.
67
+ previous_time = cron.previous_time(time).to_utc_time
68
+ previous_time + duration < time
69
+ end
70
+ end
71
+ end
72
+ end
@@ -28,10 +28,6 @@ module RocketJob
28
28
  #
29
29
  # If an exception was thrown the entire slice of records is marked as failed.
30
30
  #
31
- # If the mongo_ha gem has been loaded, then the connection to mongo is
32
- # automatically re-established and the job will resume anytime a
33
- # Mongo connection failure occurs.
34
- #
35
31
  # Thread-safe, can be called by multiple threads at the same time
36
32
  def rocket_job_work(worker, re_raise_exceptions = false)
37
33
  raise "Job must be started before calling #rocket_job_work" unless running?
@@ -50,7 +46,7 @@ module RocketJob
50
46
  next if slice.failed?
51
47
 
52
48
  slice.fail_on_exception!(re_raise_exceptions) { rocket_job_process_slice(slice) }
53
- elsif record_count && rocket_job_batch_complete?(worker.name)
49
+ elsif record_count && fail_on_exception!(re_raise_exceptions) { rocket_job_batch_complete?(worker.name) }
54
50
  return false
55
51
  else
56
52
  logger.debug "No more work available for this job"
@@ -114,8 +110,6 @@ module RocketJob
114
110
  servers
115
111
  end
116
112
 
117
- private
118
-
119
113
  def rocket_job_batch_throttled?(slice, worker)
120
114
  filter = self.class.rocket_job_batch_throttles.matching_filter(self, slice)
121
115
  return false unless filter
@@ -152,13 +146,15 @@ module RocketJob
152
146
  count = 0
153
147
  RocketJob::Sliced::Writer::Output.collect(self, slice) do |writer|
154
148
  records = slice.records
155
- slice.processing_record_number ||= 0
156
149
 
157
150
  # Skip records already processed, if any.
151
+ # slice.processing_record_number ||= 0
158
152
  # TODO: Must append to existing output slices before this can be enabled.
159
153
  # if !collect_output && (slice.processing_record_number > 1)
160
154
  # records = records[slice.processing_record_number - 1..-1]
161
155
  # end
156
+ # Until the changes above have been implemented, reprocess all records in the slice.
157
+ slice.processing_record_number = 0
162
158
 
163
159
  records.each do |record|
164
160
  slice.processing_record_number += 1
@@ -104,8 +104,6 @@ module RocketJob
104
104
  end
105
105
  end
106
106
 
107
- private
108
-
109
107
  @load_time = Time.now.utc
110
108
  @subscribers = Concurrent::Map.new { Concurrent::Array.new }
111
109
 
@@ -22,8 +22,6 @@ module RocketJob
22
22
  @collection_name = collection_name&.to_sym
23
23
  end
24
24
 
25
- private
26
-
27
25
  module ClassMethods
28
26
  def with_collection(collection_name)
29
27
  all.with_collection(collection_name)
@@ -0,0 +1,12 @@
1
+ require "mongoid/fields/validators/macro"
2
+ require "semantic_logger"
3
+ module RocketJob
4
+ module RemoveMongoidWarnings
5
+ # Remove annoying warnings about Symbols type being deprecated.
6
+ def validate_options(*params)
7
+ SemanticLogger.silence(:error) { super(*params) }
8
+ end
9
+ end
10
+ end
11
+
12
+ ::Mongoid::Fields::Validators::Macro.extend(RocketJob::RemoveMongoidWarnings)
@@ -25,7 +25,7 @@ module RocketJob
25
25
 
26
26
  self.destroy_on_complete = false
27
27
  # Number of times to automatically retry the copy. Set to `0` for no retry attempts.
28
- self.retry_limit = 5
28
+ self.retry_limit = 10
29
29
 
30
30
  # File names in IOStreams URL format.
31
31
  field :source_url, type: String, user_editable: true
@@ -1,8 +1,5 @@
1
- begin
2
- require "active_record"
3
- rescue LoadError
4
- raise 'RocketJob::Jobs::ReEncrypt::RelationalJob uses ActiveRecord to obtain the database connection, please install the gem "activerecord".'
5
- end
1
+ require "active_record"
2
+ require "sync_attr"
6
3
 
7
4
  # Batch Worker to Re-encrypt all encrypted fields in MySQL that start with `encrytped_`.
8
5
  #
@@ -1,4 +1,5 @@
1
1
  require "active_support/concern"
2
+ require "fugit"
2
3
 
3
4
  module RocketJob
4
5
  module Plugins
@@ -17,7 +18,9 @@ module RocketJob
17
18
 
18
19
  field :cron_schedule, type: String, class_attribute: true, user_editable: true, copy_on_restart: true
19
20
 
20
- validate :rocket_job_cron_valid
21
+ validates_each :cron_schedule do |record, attr, value|
22
+ record.errors.add(attr, "Invalid cron_schedule: #{value.inspect}") if value && !Fugit::Cron.new(value)
23
+ end
21
24
  before_save :rocket_job_cron_set_run_at
22
25
 
23
26
  private
@@ -42,30 +45,10 @@ module RocketJob
42
45
  end
43
46
  end
44
47
 
45
- # Returns [Time] the next time this job will be scheduled to run at.
46
- #
47
- # Parameters
48
- # time: [Time]
49
- # The next time as of this time.
50
- # Default: Time.now
51
- def rocket_job_cron_next_time(time = Time.now)
52
- RocketJob::Plugins::Rufus::CronLine.new(cron_schedule).next_time(time)
53
- end
54
-
55
- private
56
-
57
48
  def rocket_job_cron_set_run_at
58
- return unless cron_schedule
59
-
60
- self.run_at = rocket_job_cron_next_time if cron_schedule_changed? && !run_at_changed?
61
- end
62
-
63
- def rocket_job_cron_valid
64
- return unless cron_schedule
49
+ return if cron_schedule.nil? || !(cron_schedule_changed? && !run_at_changed?)
65
50
 
66
- RocketJob::Plugins::Rufus::CronLine.new(cron_schedule)
67
- rescue ArgumentError => e
68
- errors.add(:cron_schedule, e.message)
51
+ self.run_at = Fugit::Cron.new(cron_schedule).next_time.to_utc_time
69
52
  end
70
53
  end
71
54
  end
@@ -7,7 +7,7 @@ module RocketJob
7
7
  #
8
8
  # Example:
9
9
  # # Do not run this job when the MySQL slave delay exceeds 5 minutes.
10
- # class MyJob < RocketJob
10
+ # class MyJob < RocketJob::Job
11
11
  # # Define a custom mysql throttle
12
12
  # # Prevents all jobs of this class from running on the current server.
13
13
  # define_throttle :mysql_throttle_exceeded?
@@ -6,7 +6,7 @@ module RocketJob
6
6
  # Throttle the number of jobs of a specific class that are processed at the same time.
7
7
  #
8
8
  # Example:
9
- # class MyJob < RocketJob
9
+ # class MyJob < RocketJob::Job
10
10
  # # Maximum number of jobs of this class to process at the same time.
11
11
  # self.throttle_running_jobs = 25
12
12
  #
@@ -73,12 +73,13 @@ module RocketJob
73
73
  if failed? || !may_fail?
74
74
  self.exception = JobException.from_exception(e)
75
75
  exception.worker_name = worker_name
76
- save! unless new_record? || destroyed?
77
- elsif new_record? || destroyed?
78
- fail(worker_name, e)
79
76
  else
80
- fail!(worker_name, e)
77
+ fail(worker_name, e)
81
78
  end
79
+
80
+ # Prevent validation failures from failing the job
81
+ save(validate: false) unless new_record? || destroyed?
82
+
82
83
  raise e if re_raise_exceptions
83
84
  end
84
85
  end
@@ -1,4 +1,5 @@
1
1
  require "active_support/concern"
2
+ require "fugit"
2
3
 
3
4
  module RocketJob
4
5
  module Plugins
@@ -47,18 +48,14 @@ module RocketJob
47
48
 
48
49
  validates_presence_of :processing_schedule, :processing_duration
49
50
  validates_each :processing_schedule do |record, attr, value|
50
- begin
51
- RocketJob::Plugins::Rufus::CronLine.new(value)
52
- rescue ArgumentError => e
53
- record.errors.add(attr, e.message)
54
- end
51
+ record.errors.add(attr, "Invalid schedule: #{value.inspect}") unless Fugit::Cron.new(value)
55
52
  end
56
53
  end
57
54
 
58
55
  # Returns [true|false] whether this job is currently inside its processing window
59
56
  def rocket_job_processing_window_active?
60
- time = Time.now
61
- previous_time = rocket_job_processing_schedule.previous_time(time)
57
+ time = Time.now.utc
58
+ previous_time = Fugit::Cron.new(processing_schedule).previous_time(time).to_utc_time
62
59
  # Inside previous processing window?
63
60
  previous_time + processing_duration > time
64
61
  end
@@ -69,17 +66,14 @@ module RocketJob
69
66
  def rocket_job_processing_window_check
70
67
  return if rocket_job_processing_window_active?
71
68
 
72
- logger.warn("Processing window closed before job was processed. Job is re-scheduled to run at: #{rocket_job_processing_schedule.next_time}")
69
+ next_time = Fugit::Cron.new(processing_schedule).next_time.to_utc_time
70
+ logger.warn("Processing window closed before job was processed. Job is re-scheduled to run at: #{next_time}")
73
71
  self.worker_name ||= "inline"
74
72
  requeue!(worker_name)
75
73
  end
76
74
 
77
75
  def rocket_job_processing_window_set_run_at
78
- self.run_at = rocket_job_processing_schedule.next_time unless rocket_job_processing_window_active?
79
- end
80
-
81
- def rocket_job_processing_schedule
82
- RocketJob::Plugins::Rufus::CronLine.new(processing_schedule)
76
+ self.run_at = Fugit::Cron.new(processing_schedule).next_time.to_utc_time unless rocket_job_processing_window_active?
83
77
  end
84
78
  end
85
79
  end