rocketjob 6.0.0.rc2 → 6.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +164 -8
  3. data/lib/rocket_job/batch/categories.rb +26 -24
  4. data/lib/rocket_job/batch/io.rb +128 -128
  5. data/lib/rocket_job/batch/worker.rb +14 -12
  6. data/lib/rocket_job/batch.rb +0 -1
  7. data/lib/rocket_job/category/base.rb +10 -7
  8. data/lib/rocket_job/category/input.rb +61 -1
  9. data/lib/rocket_job/category/output.rb +9 -0
  10. data/lib/rocket_job/dirmon_entry.rb +1 -1
  11. data/lib/rocket_job/job_exception.rb +1 -1
  12. data/lib/rocket_job/jobs/conversion_job.rb +43 -0
  13. data/lib/rocket_job/jobs/dirmon_job.rb +24 -35
  14. data/lib/rocket_job/jobs/housekeeping_job.rb +4 -5
  15. data/lib/rocket_job/jobs/on_demand_batch_job.rb +15 -11
  16. data/lib/rocket_job/jobs/on_demand_job.rb +2 -2
  17. data/lib/rocket_job/jobs/upload_file_job.rb +4 -0
  18. data/lib/rocket_job/plugins/cron.rb +60 -20
  19. data/lib/rocket_job/plugins/job/persistence.rb +36 -0
  20. data/lib/rocket_job/plugins/restart.rb +3 -110
  21. data/lib/rocket_job/plugins/state_machine.rb +2 -2
  22. data/lib/rocket_job/plugins/throttle_dependent_jobs.rb +10 -5
  23. data/lib/rocket_job/sliced/bzip2_output_slice.rb +18 -19
  24. data/lib/rocket_job/sliced/compressed_slice.rb +3 -6
  25. data/lib/rocket_job/sliced/encrypted_bzip2_output_slice.rb +49 -0
  26. data/lib/rocket_job/sliced/encrypted_slice.rb +4 -6
  27. data/lib/rocket_job/sliced/input.rb +42 -54
  28. data/lib/rocket_job/sliced/slice.rb +7 -3
  29. data/lib/rocket_job/sliced/slices.rb +12 -9
  30. data/lib/rocket_job/sliced/writer/input.rb +46 -18
  31. data/lib/rocket_job/sliced.rb +1 -19
  32. data/lib/rocket_job/subscribers/secret_config.rb +17 -0
  33. data/lib/rocket_job/supervisor.rb +10 -8
  34. data/lib/rocket_job/version.rb +1 -1
  35. data/lib/rocketjob.rb +4 -3
  36. metadata +12 -12
  37. data/lib/rocket_job/batch/tabular/input.rb +0 -133
  38. data/lib/rocket_job/batch/tabular/output.rb +0 -67
  39. data/lib/rocket_job/batch/tabular.rb +0 -58
@@ -0,0 +1,17 @@
1
+ module RocketJob
2
+ module Subscribers
3
+ # Cause all instances to refresh their in-memory copy
4
+ # of the Secret Config Registry
5
+ #
6
+ # RocketJob::Subscribers::SecretConfig.publish(:refresh)
7
+ class SecretConfig
8
+ include RocketJob::Subscriber
9
+
10
+ def refresh
11
+ logger.measure_info "Refreshed Secret Config" do
12
+ ::SecretConfig.refresh!
13
+ end
14
+ end
15
+ end
16
+ end
17
+ end
@@ -14,6 +14,7 @@ module RocketJob
14
14
  Thread.current.name = "rocketjob main"
15
15
  RocketJob.create_indexes
16
16
  register_signal_handlers
17
+ subscribe_to_events
17
18
 
18
19
  server = Server.create!
19
20
  new(server).run
@@ -34,14 +35,8 @@ module RocketJob
34
35
  logger.info "Rocket Job Server started"
35
36
 
36
37
  event_listener = Thread.new { Event.listener }
37
- Subscribers::Server.subscribe(self) do
38
- Subscribers::Worker.subscribe(self) do
39
- Subscribers::Logger.subscribe do
40
- supervise_pool
41
- stop!
42
- end
43
- end
44
- end
38
+ supervise_pool
39
+ stop!
45
40
  rescue ::Mongoid::Errors::DocumentNotFound
46
41
  logger.info("Server has been destroyed. Going down hard!")
47
42
  rescue Exception => e
@@ -94,5 +89,12 @@ module RocketJob
94
89
  def synchronize(&block)
95
90
  @mutex.synchronize(&block)
96
91
  end
92
+
93
+ def subscribe_to_events
94
+ Subscribers::Logger.subscribe
95
+ Subscribers::SecretConfig.subscribe if defined?(SecretConfig)
96
+ Subscribers::Server.subscribe
97
+ Subscribers::Worker.subscribe
98
+ end
97
99
  end
98
100
  end
@@ -1,3 +1,3 @@
1
1
  module RocketJob
2
- VERSION = "6.0.0.rc2".freeze
2
+ VERSION = "6.0.2".freeze
3
3
  end
data/lib/rocketjob.rb CHANGED
@@ -63,7 +63,6 @@ module RocketJob
63
63
  autoload :Cron, "rocket_job/plugins/cron"
64
64
  autoload :Document, "rocket_job/plugins/document"
65
65
  autoload :ProcessingWindow, "rocket_job/plugins/processing_window"
66
- autoload :Restart, "rocket_job/plugins/restart"
67
66
  autoload :Retry, "rocket_job/plugins/retry"
68
67
  autoload :Singleton, "rocket_job/plugins/singleton"
69
68
  autoload :StateMachine, "rocket_job/plugins/state_machine"
@@ -73,22 +72,24 @@ module RocketJob
73
72
 
74
73
  module Jobs
75
74
  autoload :ActiveJob, "rocket_job/jobs/active_job"
75
+ autoload :ConversionJob, "rocket_job/jobs/conversion_job"
76
76
  autoload :CopyFileJob, "rocket_job/jobs/copy_file_job"
77
77
  autoload :DirmonJob, "rocket_job/jobs/dirmon_job"
78
+ autoload :HousekeepingJob, "rocket_job/jobs/housekeeping_job"
78
79
  autoload :OnDemandBatchJob, "rocket_job/jobs/on_demand_batch_job"
79
80
  autoload :OnDemandJob, "rocket_job/jobs/on_demand_job"
80
- autoload :HousekeepingJob, "rocket_job/jobs/housekeeping_job"
81
81
  autoload :PerformanceJob, "rocket_job/jobs/performance_job"
82
82
  autoload :SimpleJob, "rocket_job/jobs/simple_job"
83
83
  autoload :UploadFileJob, "rocket_job/jobs/upload_file_job"
84
84
 
85
85
  module ReEncrypt
86
- autoload :RelationalJob, "rocket_job/jobs/re_encrypt/relational_job"
86
+ autoload :RelationalJob, "rocket_job/jobs/re_encrypt/relational_job"
87
87
  end
88
88
  end
89
89
 
90
90
  module Subscribers
91
91
  autoload :Logger, "rocket_job/subscribers/logger"
92
+ autoload :SecretConfig, "rocket_job/subscribers/secret_config"
92
93
  autoload :Server, "rocket_job/subscribers/server"
93
94
  autoload :Worker, "rocket_job/subscribers/worker"
94
95
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rocketjob
3
3
  version: !ruby/object:Gem::Version
4
- version: 6.0.0.rc2
4
+ version: 6.0.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Reid Morrison
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-06-16 00:00:00.000000000 Z
11
+ date: 2021-10-25 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: aasm
@@ -58,14 +58,14 @@ dependencies:
58
58
  requirements:
59
59
  - - "~>"
60
60
  - !ruby/object:Gem::Version
61
- version: '1.6'
61
+ version: '1.9'
62
62
  type: :runtime
63
63
  prerelease: false
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
66
  - - "~>"
67
67
  - !ruby/object:Gem::Version
68
- version: '1.6'
68
+ version: '1.9'
69
69
  - !ruby/object:Gem::Dependency
70
70
  name: mongoid
71
71
  requirement: !ruby/object:Gem::Requirement
@@ -84,14 +84,14 @@ dependencies:
84
84
  name: semantic_logger
85
85
  requirement: !ruby/object:Gem::Requirement
86
86
  requirements:
87
- - - "~>"
87
+ - - ">="
88
88
  - !ruby/object:Gem::Version
89
89
  version: '4.7'
90
90
  type: :runtime
91
91
  prerelease: false
92
92
  version_requirements: !ruby/object:Gem::Requirement
93
93
  requirements:
94
- - - "~>"
94
+ - - ">="
95
95
  - !ruby/object:Gem::Version
96
96
  version: '4.7'
97
97
  - !ruby/object:Gem::Dependency
@@ -134,9 +134,6 @@ files:
134
134
  - lib/rocket_job/batch/results.rb
135
135
  - lib/rocket_job/batch/state_machine.rb
136
136
  - lib/rocket_job/batch/statistics.rb
137
- - lib/rocket_job/batch/tabular.rb
138
- - lib/rocket_job/batch/tabular/input.rb
139
- - lib/rocket_job/batch/tabular/output.rb
140
137
  - lib/rocket_job/batch/throttle.rb
141
138
  - lib/rocket_job/batch/throttle_running_workers.rb
142
139
  - lib/rocket_job/batch/throttle_windows.rb
@@ -160,6 +157,7 @@ files:
160
157
  - lib/rocket_job/job.rb
161
158
  - lib/rocket_job/job_exception.rb
162
159
  - lib/rocket_job/jobs/active_job.rb
160
+ - lib/rocket_job/jobs/conversion_job.rb
163
161
  - lib/rocket_job/jobs/copy_file_job.rb
164
162
  - lib/rocket_job/jobs/dirmon_job.rb
165
163
  - lib/rocket_job/jobs/housekeeping_job.rb
@@ -197,6 +195,7 @@ files:
197
195
  - lib/rocket_job/sliced.rb
198
196
  - lib/rocket_job/sliced/bzip2_output_slice.rb
199
197
  - lib/rocket_job/sliced/compressed_slice.rb
198
+ - lib/rocket_job/sliced/encrypted_bzip2_output_slice.rb
200
199
  - lib/rocket_job/sliced/encrypted_slice.rb
201
200
  - lib/rocket_job/sliced/input.rb
202
201
  - lib/rocket_job/sliced/output.rb
@@ -206,6 +205,7 @@ files:
206
205
  - lib/rocket_job/sliced/writer/output.rb
207
206
  - lib/rocket_job/subscriber.rb
208
207
  - lib/rocket_job/subscribers/logger.rb
208
+ - lib/rocket_job/subscribers/secret_config.rb
209
209
  - lib/rocket_job/subscribers/server.rb
210
210
  - lib/rocket_job/subscribers/worker.rb
211
211
  - lib/rocket_job/supervisor.rb
@@ -232,11 +232,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
232
232
  version: '2.5'
233
233
  required_rubygems_version: !ruby/object:Gem::Requirement
234
234
  requirements:
235
- - - ">"
235
+ - - ">="
236
236
  - !ruby/object:Gem::Version
237
- version: 1.3.1
237
+ version: '0'
238
238
  requirements: []
239
- rubygems_version: 3.2.15
239
+ rubygems_version: 3.2.22
240
240
  signing_key:
241
241
  specification_version: 4
242
242
  summary: Ruby's missing batch processing system.
@@ -1,133 +0,0 @@
1
- require "active_support/concern"
2
-
3
- module RocketJob
4
- module Batch
5
- class Tabular
6
- # @deprecated
7
- module Input
8
- extend ActiveSupport::Concern
9
-
10
- included do
11
- warn "#{name} is using RocketJob::Batch::Tabular::Input which is deprecated"
12
-
13
- field :tabular_input_header, type: Array, class_attribute: true, user_editable: true
14
- field :tabular_input_format, type: Mongoid::StringifiedSymbol, default: :csv, class_attribute: true, user_editable: true
15
- field :tabular_input_options, type: Hash, class_attribute: true
16
-
17
- # tabular_input_mode: [:line | :array | :hash]
18
- # :line
19
- # Uploads the file a line (String) at a time for processing by workers.
20
- # :array
21
- # Parses each line from the file as an Array and uploads each array for processing by workers.
22
- # :hash
23
- # Parses each line from the file into a Hash and uploads each hash for processing by workers.
24
- # See IOStreams#each.
25
- field :tabular_input_mode, type: Mongoid::StringifiedSymbol, default: :line, class_attribute: true, user_editable: true, copy_on_restart: true
26
-
27
- validates_inclusion_of :tabular_input_format, in: IOStreams::Tabular.registered_formats
28
- validates_inclusion_of :tabular_input_mode, in: %i[line array hash row record]
29
- validate :tabular_input_header_present
30
-
31
- class_attribute :tabular_input_white_list
32
- class_attribute :tabular_input_required
33
- class_attribute :tabular_input_skip_unknown
34
-
35
- # Cleanse all uploaded data by removing non-printable characters
36
- # and any characters that cannot be converted to UTF-8
37
- class_attribute :tabular_input_type
38
-
39
- self.tabular_input_white_list = nil
40
- self.tabular_input_required = nil
41
- self.tabular_input_skip_unknown = true
42
- self.tabular_input_type = :text
43
-
44
- before_perform :tabular_input_render
45
- end
46
-
47
- # Extract the header line during the upload.
48
- #
49
- # Overrides: RocketJob::Batch::IO#upload
50
- #
51
- # Notes:
52
- # - When supplying a block the header must be set manually
53
- def upload(stream = nil, **args, &block)
54
- input_stream = stream.nil? ? nil : IOStreams.new(stream)
55
-
56
- if stream && (tabular_input_type == :text)
57
- # Cannot change the length of fixed width lines
58
- replace = tabular_input_format == :fixed ? " " : ""
59
- input_stream.option_or_stream(:encode, encoding: "UTF-8", cleaner: :printable, replace: replace)
60
- end
61
-
62
- # If an input header is not required, then we don't extract it'
63
- return super(input_stream, stream_mode: tabular_input_mode, **args, &block) unless tabular_input.header?
64
-
65
- # If the header is already set then it is not expected in the file
66
- if tabular_input_header.present?
67
- tabular_input_cleanse_header
68
- return super(input_stream, stream_mode: tabular_input_mode, **args, &block)
69
- end
70
-
71
- case tabular_input_mode
72
- when :line
73
- parse_header = lambda do |line|
74
- tabular_input.parse_header(line)
75
- tabular_input_cleanse_header
76
- self.tabular_input_header = tabular_input.header.columns
77
- end
78
- super(input_stream, on_first: parse_header, stream_mode: :line, **args, &block)
79
- when :array, :row
80
- set_header = lambda do |row|
81
- tabular_input.header.columns = row
82
- tabular_input_cleanse_header
83
- self.tabular_input_header = tabular_input.header.columns
84
- end
85
- super(input_stream, on_first: set_header, stream_mode: :array, **args, &block)
86
- when :hash, :record
87
- super(input_stream, stream_mode: :hash, **args, &block)
88
- else
89
- raise(ArgumentError, "Invalid tabular_input_mode: #{stream_mode.inspect}")
90
- end
91
- end
92
-
93
- private
94
-
95
- # Shared instance used for this slice, by a single worker (thread)
96
- def tabular_input
97
- @tabular_input ||= IOStreams::Tabular.new(
98
- columns: tabular_input_header,
99
- allowed_columns: tabular_input_white_list,
100
- required_columns: tabular_input_required,
101
- skip_unknown: tabular_input_skip_unknown,
102
- format: tabular_input_format,
103
- format_options: tabular_input_options&.deep_symbolize_keys
104
- )
105
- end
106
-
107
- def tabular_input_render
108
- return if tabular_input_header.blank? && tabular_input.header?
109
-
110
- @rocket_job_input = tabular_input.record_parse(@rocket_job_input)
111
- end
112
-
113
- # Cleanse custom input header if supplied.
114
- def tabular_input_cleanse_header
115
- ignored_columns = tabular_input.header.cleanse!
116
- logger.warn("Stripped out invalid columns from custom header", ignored_columns) unless ignored_columns.empty?
117
-
118
- self.tabular_input_header = tabular_input.header.columns
119
- end
120
-
121
- def tabular_input_header_present
122
- if tabular_input_header.present? ||
123
- !tabular_input.header? ||
124
- (tabular_input_mode == :hash || tabular_input_mode == :record)
125
- return
126
- end
127
-
128
- errors.add(:tabular_input_header, "is required when tabular_input_format is #{tabular_input_format.inspect}")
129
- end
130
- end
131
- end
132
- end
133
- end
@@ -1,67 +0,0 @@
1
- require "active_support/concern"
2
-
3
- module RocketJob
4
- module Batch
5
- class Tabular
6
- # For the simple case where all `output_categories` have the same format,
7
- # If multiple output categories are used with different formats, then use IOStreams::Tabular directly
8
- # instead of this plugin.
9
- module Output
10
- extend ActiveSupport::Concern
11
-
12
- included do
13
- warn "#{name} is using RocketJob::Batch::Tabular::Output which is deprecated"
14
-
15
- field :tabular_output_header, type: Array, class_attribute: true, user_editable: true, copy_on_restart: true
16
- field :tabular_output_format, type: Mongoid::StringifiedSymbol, default: :csv, class_attribute: true, user_editable: true, copy_on_restart: true
17
- field :tabular_output_options, type: Hash, class_attribute: true
18
-
19
- validates_inclusion_of :tabular_output_format, in: IOStreams::Tabular.registered_formats
20
-
21
- after_perform :tabular_output_render
22
- end
23
-
24
- # Clear out cached tabular_output any time header or format is changed.
25
- def tabular_output_header=(tabular_output_header)
26
- super(tabular_output_header)
27
- @tabular_output = nil
28
- end
29
-
30
- def tabular_output_format=(tabular_output_format)
31
- super(tabular_output_format)
32
- @tabular_output = nil
33
- end
34
-
35
- # Overrides: `RocketJob::Batch::IO#download` to add the `tabular_output_header`.
36
- def download(file_name_or_io = nil, category: :main, **args, &block)
37
- unless tabular_output.requires_header?(category)
38
- return super(file_name_or_io, category: category, **args, &block)
39
- end
40
-
41
- header = tabular_output.render_header(category)
42
- super(file_name_or_io, header_line: header, category: category, **args, &block)
43
- end
44
-
45
- private
46
-
47
- # Delimited instance used for this slice, by a single worker (thread)
48
- def tabular_output
49
- @tabular_output ||= Tabular.new(
50
- main: IOStreams::Tabular.new(
51
- columns: tabular_output_header,
52
- format: tabular_output_format,
53
- format_options: tabular_output_options&.deep_symbolize_keys
54
- )
55
- )
56
- end
57
-
58
- # Render the output from the perform.
59
- def tabular_output_render
60
- return unless output_categories.present?
61
-
62
- @rocket_job_output = tabular_output.render(@rocket_job_output)
63
- end
64
- end
65
- end
66
- end
67
- end
@@ -1,58 +0,0 @@
1
- module RocketJob
2
- module Batch
3
- # Format output results.
4
- #
5
- # Takes Batch::Results, Batch::Result, Hash, Array, or String and renders it for output.
6
- #
7
- # Example:
8
- #
9
- # tabular = Tabular.new(
10
- # main: IOStreams::Tabular.new(columns: main_file_headers, format: tabular_output_format),
11
- # exceptions: IOStreams::Tabular.new(columns: exception_file_headers, format: tabular_output_format)
12
- # )
13
- #
14
- # tabular.render(row)
15
- #
16
- # @deprecated
17
- class Tabular
18
- autoload :Input, "rocket_job/batch/tabular/input"
19
- autoload :Output, "rocket_job/batch/tabular/output"
20
-
21
- def initialize(map)
22
- @map = map
23
- end
24
-
25
- def [](category = :main)
26
- @map[category] || raise("No tabular map defined for category: #{category.inspect}")
27
- end
28
-
29
- # Iterate over responses and format using Tabular
30
- def render(row, category = :main)
31
- if row.is_a?(Batch::Results)
32
- results = Batch::Results.new
33
- row.each { |result| results << render(result) }
34
- results
35
- elsif row.is_a?(Batch::Result)
36
- row.value = self[row.category].render(row.value)
37
- row
38
- elsif row.blank?
39
- nil
40
- else
41
- self[category].render(row)
42
- end
43
- end
44
-
45
- def render_header(category = :main)
46
- self[category].render_header
47
- end
48
-
49
- def requires_header?(category = :main)
50
- self[category].requires_header?
51
- end
52
-
53
- def header?(category = :main)
54
- self[category].header?
55
- end
56
- end
57
- end
58
- end